diff --git a/src/main/java/org/cloudsimplus/examples/traces/google/GoogleMachineAndTaskEventsExample1.java b/src/main/java/org/cloudsimplus/examples/traces/google/GoogleMachineAndTaskEventsExample1.java new file mode 100644 index 0000000..829cf27 --- /dev/null +++ b/src/main/java/org/cloudsimplus/examples/traces/google/GoogleMachineAndTaskEventsExample1.java @@ -0,0 +1,169 @@ +/* + * CloudSim Plus: A modern, highly-extensible and easier-to-use Framework for + * Modeling and Simulation of Cloud Computing Infrastructures and Services. + * http://cloudsimplus.org + * + * Copyright (C) 2015-2021 Universidade da Beira Interior (UBI, Portugal) and + * the Instituto Federal de Educação Ciência e Tecnologia do Tocantins (IFTO, Brazil). + * + * This file is part of CloudSim Plus. + * + * CloudSim Plus is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CloudSim Plus is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CloudSim Plus. If not, see . + */ +package org.cloudsimplus.examples.traces.google; + +import org.cloudsimplus.brokers.DatacenterBroker; +import org.cloudsimplus.builders.tables.CloudletsTableBuilder; +import org.cloudsimplus.builders.tables.TextTableColumn; +import org.cloudsimplus.cloudlets.Cloudlet; +import org.cloudsimplus.cloudlets.CloudletSimple; +import org.cloudsimplus.core.CloudSimPlus; +import org.cloudsimplus.datacenters.Datacenter; +import org.cloudsimplus.datacenters.DatacenterSimple; +import org.cloudsimplus.hosts.Host; +import org.cloudsimplus.hosts.HostSimple; +import org.cloudsimplus.placementpolicies.VmPlacementPolicySimple; +import org.cloudsimplus.resources.CoreCapacity; +import org.cloudsimplus.schedulers.vm.VmSchedulerTimeShared; +import org.cloudsimplus.traces.google.GoogleMachineEventsTraceReader; +import org.cloudsimplus.traces.google.GoogleTaskEventsTraceReader; +import org.cloudsimplus.traces.google.MachineEvent; +import org.cloudsimplus.traces.google.TaskEvent; +import org.cloudsimplus.traces.google.TaskVmRequest; +import org.cloudsimplus.utilizationmodels.UtilizationModelDynamic; +import org.cloudsimplus.utilizationmodels.UtilizationModelFull; +import org.cloudsimplus.vms.Vm; +import org.cloudsimplus.vms.VmSimple; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; + +/// An example that creates both Hosts and Cloudlets from +/// [Google Cluster Data](https://github.com/google/cluster-data) trace files: +/// Hosts come from a "machine events" trace and Cloudlets from a "task events" trace. +/// +/// The machine ID in the task events defines where each task runs: +/// the tasks of a user in the same machine run inside the same VM, which is placed in the Host with that ID. +/// The trace doesn't define VMs, so they are created by the [#createVm(TaskVmRequest)] method. +/// You can confirm that in the "Host" and "VM" columns of the results. +/// +/// The machine IDs in the trace files must match. That's why the machine events +/// sample used here is different from the one in the [GoogleMachineEventsExample1]. +/// Tasks without a machine ID in the trace run in any VM of their user. +/// +/// @author Manoel Campos da Silva Filho +/// @since CloudSim Plus 9.0.0 +public class GoogleMachineAndTaskEventsExample1 { + private static final String MACHINE_EVENTS_FILE = "workload/google-traces/machine-events-sample-2.csv"; + private static final String TASK_EVENTS_FILE = "workload/google-traces/task-events-sample-1.csv"; + + /// The capacity of the largest machine, which the normalized values in the trace files are relative to. + private static final int MAX_CORES = 8; + private static final long MAX_RAM = 16384; //in Megabytes + private static final long HOST_STORAGE = 1000000; //in Megabytes + private static final long HOST_BW = 10000; //in Megabits/s + private static final double MIPS = 1000; + + private static final long VM_BW = 100; //in Megabits/s + private static final long VM_SIZE = 10000; //in Megabytes + + /// Defines a negative length for Cloudlets, so that they run until the trace says they finished. + private static final int CLOUDLET_LENGTH = -10_000; + private static final long CLOUDLET_FILE_SIZE = 1024; //in bytes + + private final CloudSimPlus simulation; + + public static void main(String[] args) { + new GoogleMachineAndTaskEventsExample1(); + } + + private GoogleMachineAndTaskEventsExample1() { + simulation = new CloudSimPlus(); + final Datacenter datacenter = createDatacenterFromTraceFile(); + final var taskReader = createTaskEventsReader(datacenter); + taskReader.process(); + + simulation.start(); + + taskReader.getBrokerManager().getBrokers().stream().sorted().forEach(this::printCloudlets); + } + + /// Creates a Datacenter with the Hosts defined in the machine events trace. + /// Hosts added during the simulation would be added to the same Datacenter. + private Datacenter createDatacenterFromTraceFile() { + final var datacenter = DatacenterSimple.of(simulation, new VmPlacementPolicySimple()); + final var reader = GoogleMachineEventsTraceReader.getInstance(MACHINE_EVENTS_FILE, this::createHost); + reader.setMaxCpuCores(MAX_CORES); + reader.setMaxRamCapacity(MAX_RAM); + reader.setDatacenterForLaterHosts(datacenter); + datacenter.addHostList(new ArrayList<>(reader.process())); + return datacenter; + } + + /// Creates a reader that runs tasks in the Hosts given by the trace. + private GoogleTaskEventsTraceReader createTaskEventsReader(final Datacenter datacenter) { + final var reader = + GoogleTaskEventsTraceReader + .getInstance(simulation, TASK_EVENTS_FILE, this::createCloudlet) + .mapTasksToHosts(this::createVm); + + //Makes the Datacenter place the VMs created by the reader in the Host with their machine ID + datacenter.getVmPlacementPolicy().setFindHostForVmFunction(reader::findHostForVm); + return reader; + } + + private Host createHost(final MachineEvent event) { + final Host host = HostSimple.of(event.getRam(), HOST_BW, HOST_STORAGE, new CoreCapacity(event.getCpuCores(), MIPS)); + host.setVmScheduler(new VmSchedulerTimeShared()); + return host; + } + + /// Creates the VM that runs the tasks of a user in a machine. + /// You can write your own code here to define how the VM is sized from the trace data. + /// The request provides the peak of resources requested by the tasks running at the same time. + /// + /// The VM size doesn't use the peak of local disk space, because requests in the trace + /// can exceed the machine capacity. In the sample trace, they exceed it in machine 1, + /// which would make the Host refuse the VM of the second user. + /// + /// @param request the data about the tasks of a user in a machine + /// @return the created VM + private Vm createVm(final TaskVmRequest request) { + final long cores = Math.max(1, (long)Math.ceil(request.peakCpuCores() * MAX_CORES)); + final long ram = Math.max(1, (long)Math.ceil(request.peakRam() * MAX_RAM)); + return VmSimple.of(new CoreCapacity(cores, MIPS)).setRam(ram).setBw(VM_BW).setSize(VM_SIZE); + } + + /// Creates a Cloudlet from a task in the trace. + /// The RAM [UtilizationModelDynamic] is required by the reader, since the max RAM usage may change in the trace. + private Cloudlet createCloudlet(final TaskEvent event) { + final long cores = Math.max(1, event.actualCpuCores(MAX_CORES)); + return CloudletSimple.of(CLOUDLET_LENGTH, cores) + .setFileSize(CLOUDLET_FILE_SIZE) + .setOutputSize(CLOUDLET_FILE_SIZE) + .setUtilizationModelCpu(new UtilizationModelFull()) + .setUtilizationModelBw(new UtilizationModelDynamic(0.25)) + .setUtilizationModelRam(new UtilizationModelDynamic(0, event.getResourceRequestForRam())); + } + + private void printCloudlets(final DatacenterBroker broker) { + final Collection finished = broker.getCloudletFinishedList(); + final var cloudlets = finished.stream().sorted(Comparator.comparingLong(Cloudlet::getId)).toList(); + new CloudletsTableBuilder(cloudlets) + .addColumn(new TextTableColumn("Job", "ID"), Cloudlet::getJobId, 0) + .setTitle("Simulation results for " + broker.getName()) + .build(); + } +} diff --git a/src/main/java/org/cloudsimplus/examples/traces/google/GoogleMachineEventsExample1.java b/src/main/java/org/cloudsimplus/examples/traces/google/GoogleMachineEventsExample1.java index 8527825..b8f672f 100755 --- a/src/main/java/org/cloudsimplus/examples/traces/google/GoogleMachineEventsExample1.java +++ b/src/main/java/org/cloudsimplus/examples/traces/google/GoogleMachineEventsExample1.java @@ -41,7 +41,6 @@ import org.cloudsimplus.traces.TraceReaderAbstract; import org.cloudsimplus.traces.google.GoogleMachineEventsTraceReader; import org.cloudsimplus.traces.google.MachineEvent; -import org.cloudsimplus.traces.google.MachineEventType; import org.cloudsimplus.utilizationmodels.UtilizationModelFull; import org.cloudsimplus.vms.Vm; import org.cloudsimplus.vms.VmSimple; @@ -62,15 +61,16 @@ /// as specified in the trace. CloudSim Plus accordingly process such events. You /// can confirm in the logs that Host 3 is added and removed multiple times. /// +/// The trace also has an event that changes the capacity of Host 20, +/// which ends up with more CPU cores than the others. +/// /// Check important details at [TraceReaderAbstract]. To better understand /// the structure of trace files, check the google-cluster-data-samples.xlsx /// spreadsheet inside the docs dir. /// /// @author Manoel Campos da Silva Filho /// @since CloudSim Plus 4.0.0 -/// -/// TODO [MachineEventType#UPDATE] events are not being processed yet. -/// See https://github.com/cloudsimplus/cloudsimplus/issues/590 +/// @see GoogleMachineAndTaskEventsExample1 public class GoogleMachineEventsExample1 { private static final String TRACE_FILENAME = "workload/google-traces/machine-events-sample-1.csv"; private static final int HOST_BW = 10; diff --git a/src/main/java/org/cloudsimplus/examples/traces/google/GoogleTaskEventsExample1.java b/src/main/java/org/cloudsimplus/examples/traces/google/GoogleTaskEventsExample1.java index 1a66899..3e2d85b 100755 --- a/src/main/java/org/cloudsimplus/examples/traces/google/GoogleTaskEventsExample1.java +++ b/src/main/java/org/cloudsimplus/examples/traces/google/GoogleTaskEventsExample1.java @@ -78,12 +78,7 @@ /// /// @author Manoel Campos da Silva Filho /// @since CloudSim Plus 4.0.0 -/// -/// TODO A joint example that creates Hosts and Cloudlets from trace files will be useful. -/// TODO See https://github.com/manoelcampos/cloudsimplus/issues/151 -/// TODO [CloudSimTag#CLOUDLET_FAIL] events aren't been processed. -/// TODO It has to be checked how to make the Cloudlet to be executed in the Host specified in the trace file. -/// See https://github.com/cloudsimplus/cloudsimplus/issues/590 +/// @see GoogleMachineAndTaskEventsExample1 public class GoogleTaskEventsExample1 { private static final String TASK_EVENTS_FILE = "workload/google-traces/task-events-sample-1.csv"; private static final String TASK_USAGE_FILE = "workload/google-traces/task-usage-sample-1.csv"; diff --git a/src/main/resources/workload/google-traces/machine-events-sample-1.csv b/src/main/resources/workload/google-traces/machine-events-sample-1.csv index fac2dfc..224b80d 100755 --- a/src/main/resources/workload/google-traces/machine-events-sample-1.csv +++ b/src/main/resources/workload/google-traces/machine-events-sample-1.csv @@ -5,5 +5,6 @@ 0,40,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493 10000,50,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493 10000,60,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493 +20000,20,2,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.7,0.2493 30000,30,1,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493 50000,30,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493 \ No newline at end of file diff --git a/src/main/resources/workload/google-traces/machine-events-sample-2.csv b/src/main/resources/workload/google-traces/machine-events-sample-2.csv new file mode 100644 index 0000000..33b73b0 --- /dev/null +++ b/src/main/resources/workload/google-traces/machine-events-sample-2.csv @@ -0,0 +1,6 @@ +#Time stamp (microseconds),Machine ID,Event type (0-ADD 1-REMOVE 2-UPDATE),Platform ID,capacity:CPU (%),capacity:memory (%) +0,0,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,1 +0,1,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,1 +0,2,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,1 +0,3,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,1 +40000000,1,2,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,0.75