Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
/*
* CloudSim Plus: A modern, highly-extensible and easier-to-use Framework for
* Modeling and Simulation of Cloud Computing Infrastructures and Services.
* http://cloudsimplus.org
*
* Copyright (C) 2015-2021 Universidade da Beira Interior (UBI, Portugal) and
* the Instituto Federal de Educação Ciência e Tecnologia do Tocantins (IFTO, Brazil).
*
* This file is part of CloudSim Plus.
*
* CloudSim Plus is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* CloudSim Plus is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with CloudSim Plus. If not, see <http://www.gnu.org/licenses/>.
*/
package org.cloudsimplus.examples.traces.google;

import org.cloudsimplus.brokers.DatacenterBroker;
import org.cloudsimplus.builders.tables.CloudletsTableBuilder;
import org.cloudsimplus.builders.tables.TextTableColumn;
import org.cloudsimplus.cloudlets.Cloudlet;
import org.cloudsimplus.cloudlets.CloudletSimple;
import org.cloudsimplus.core.CloudSimPlus;
import org.cloudsimplus.datacenters.Datacenter;
import org.cloudsimplus.datacenters.DatacenterSimple;
import org.cloudsimplus.hosts.Host;
import org.cloudsimplus.hosts.HostSimple;
import org.cloudsimplus.placementpolicies.VmPlacementPolicySimple;
import org.cloudsimplus.resources.CoreCapacity;
import org.cloudsimplus.schedulers.vm.VmSchedulerTimeShared;
import org.cloudsimplus.traces.google.GoogleMachineEventsTraceReader;
import org.cloudsimplus.traces.google.GoogleTaskEventsTraceReader;
import org.cloudsimplus.traces.google.MachineEvent;
import org.cloudsimplus.traces.google.TaskEvent;
import org.cloudsimplus.traces.google.TaskVmRequest;
import org.cloudsimplus.utilizationmodels.UtilizationModelDynamic;
import org.cloudsimplus.utilizationmodels.UtilizationModelFull;
import org.cloudsimplus.vms.Vm;
import org.cloudsimplus.vms.VmSimple;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;

/// An example that creates both Hosts and Cloudlets from
/// [Google Cluster Data](https://github.com/google/cluster-data) trace files:
/// Hosts come from a "machine events" trace and Cloudlets from a "task events" trace.
///
/// The machine ID in the task events defines where each task runs:
/// the tasks of a user in the same machine run inside the same VM, which is placed in the Host with that ID.
/// The trace doesn't define VMs, so they are created by the [#createVm(TaskVmRequest)] method.
/// You can confirm that in the "Host" and "VM" columns of the results.
///
/// The machine IDs in the trace files must match. That's why the machine events
/// sample used here is different from the one in the [GoogleMachineEventsExample1].
/// Tasks without a machine ID in the trace run in any VM of their user.
///
/// @author Manoel Campos da Silva Filho
/// @since CloudSim Plus 9.0.0
public class GoogleMachineAndTaskEventsExample1 {
private static final String MACHINE_EVENTS_FILE = "workload/google-traces/machine-events-sample-2.csv";
private static final String TASK_EVENTS_FILE = "workload/google-traces/task-events-sample-1.csv";

/// The capacity of the largest machine, which the normalized values in the trace files are relative to.
private static final int MAX_CORES = 8;
private static final long MAX_RAM = 16384; //in Megabytes
private static final long HOST_STORAGE = 1000000; //in Megabytes
private static final long HOST_BW = 10000; //in Megabits/s
private static final double MIPS = 1000;

private static final long VM_BW = 100; //in Megabits/s
private static final long VM_SIZE = 10000; //in Megabytes

/// Defines a negative length for Cloudlets, so that they run until the trace says they finished.
private static final int CLOUDLET_LENGTH = -10_000;
private static final long CLOUDLET_FILE_SIZE = 1024; //in bytes

private final CloudSimPlus simulation;

public static void main(String[] args) {
new GoogleMachineAndTaskEventsExample1();
}

private GoogleMachineAndTaskEventsExample1() {
simulation = new CloudSimPlus();
final Datacenter datacenter = createDatacenterFromTraceFile();
final var taskReader = createTaskEventsReader(datacenter);
taskReader.process();

simulation.start();

taskReader.getBrokerManager().getBrokers().stream().sorted().forEach(this::printCloudlets);
}

/// Creates a Datacenter with the Hosts defined in the machine events trace.
/// Hosts added during the simulation would be added to the same Datacenter.
private Datacenter createDatacenterFromTraceFile() {
final var datacenter = DatacenterSimple.of(simulation, new VmPlacementPolicySimple());
final var reader = GoogleMachineEventsTraceReader.getInstance(MACHINE_EVENTS_FILE, this::createHost);
reader.setMaxCpuCores(MAX_CORES);
reader.setMaxRamCapacity(MAX_RAM);
reader.setDatacenterForLaterHosts(datacenter);
datacenter.addHostList(new ArrayList<>(reader.process()));
return datacenter;
}

/// Creates a reader that runs tasks in the Hosts given by the trace.
private GoogleTaskEventsTraceReader createTaskEventsReader(final Datacenter datacenter) {
final var reader =
GoogleTaskEventsTraceReader
.getInstance(simulation, TASK_EVENTS_FILE, this::createCloudlet)
.mapTasksToHosts(this::createVm);

//Makes the Datacenter place the VMs created by the reader in the Host with their machine ID
datacenter.getVmPlacementPolicy().setFindHostForVmFunction(reader::findHostForVm);
return reader;
}

private Host createHost(final MachineEvent event) {
final Host host = HostSimple.of(event.getRam(), HOST_BW, HOST_STORAGE, new CoreCapacity(event.getCpuCores(), MIPS));
host.setVmScheduler(new VmSchedulerTimeShared());
return host;
}

/// Creates the VM that runs the tasks of a user in a machine.
/// You can write your own code here to define how the VM is sized from the trace data.
/// The request provides the peak of resources requested by the tasks running at the same time.
///
/// The VM size doesn't use the peak of local disk space, because requests in the trace
/// can exceed the machine capacity. In the sample trace, they exceed it in machine 1,
/// which would make the Host refuse the VM of the second user.
///
/// @param request the data about the tasks of a user in a machine
/// @return the created VM
private Vm createVm(final TaskVmRequest request) {
final long cores = Math.max(1, (long)Math.ceil(request.peakCpuCores() * MAX_CORES));
final long ram = Math.max(1, (long)Math.ceil(request.peakRam() * MAX_RAM));
return VmSimple.of(new CoreCapacity(cores, MIPS)).setRam(ram).setBw(VM_BW).setSize(VM_SIZE);
}

/// Creates a Cloudlet from a task in the trace.
/// The RAM [UtilizationModelDynamic] is required by the reader, since the max RAM usage may change in the trace.
private Cloudlet createCloudlet(final TaskEvent event) {
final long cores = Math.max(1, event.actualCpuCores(MAX_CORES));
return CloudletSimple.of(CLOUDLET_LENGTH, cores)
.setFileSize(CLOUDLET_FILE_SIZE)
.setOutputSize(CLOUDLET_FILE_SIZE)
.setUtilizationModelCpu(new UtilizationModelFull())
.setUtilizationModelBw(new UtilizationModelDynamic(0.25))
.setUtilizationModelRam(new UtilizationModelDynamic(0, event.getResourceRequestForRam()));
}

private void printCloudlets(final DatacenterBroker broker) {
final Collection<Cloudlet> finished = broker.getCloudletFinishedList();
final var cloudlets = finished.stream().sorted(Comparator.comparingLong(Cloudlet::getId)).toList();
new CloudletsTableBuilder(cloudlets)
.addColumn(new TextTableColumn("Job", "ID"), Cloudlet::getJobId, 0)
.setTitle("Simulation results for " + broker.getName())
.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import org.cloudsimplus.traces.TraceReaderAbstract;
import org.cloudsimplus.traces.google.GoogleMachineEventsTraceReader;
import org.cloudsimplus.traces.google.MachineEvent;
import org.cloudsimplus.traces.google.MachineEventType;
import org.cloudsimplus.utilizationmodels.UtilizationModelFull;
import org.cloudsimplus.vms.Vm;
import org.cloudsimplus.vms.VmSimple;
Expand All @@ -62,15 +61,16 @@
/// as specified in the trace. CloudSim Plus accordingly process such events. You
/// can confirm in the logs that Host 3 is added and removed multiple times.
///
/// The trace also has an event that changes the capacity of Host 20,
/// which ends up with more CPU cores than the others.
///
/// Check important details at [TraceReaderAbstract]. To better understand
/// the structure of trace files, check the google-cluster-data-samples.xlsx
/// spreadsheet inside the docs dir.
///
/// @author Manoel Campos da Silva Filho
/// @since CloudSim Plus 4.0.0
///
/// TODO [MachineEventType#UPDATE] events are not being processed yet.
/// See https://github.com/cloudsimplus/cloudsimplus/issues/590
/// @see GoogleMachineAndTaskEventsExample1
public class GoogleMachineEventsExample1 {
private static final String TRACE_FILENAME = "workload/google-traces/machine-events-sample-1.csv";
private static final int HOST_BW = 10;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,7 @@
///
/// @author Manoel Campos da Silva Filho
/// @since CloudSim Plus 4.0.0
///
/// TODO A joint example that creates Hosts and Cloudlets from trace files will be useful.
/// TODO See https://github.com/manoelcampos/cloudsimplus/issues/151
/// TODO [CloudSimTag#CLOUDLET_FAIL] events aren't been processed.
/// TODO It has to be checked how to make the Cloudlet to be executed in the Host specified in the trace file.
/// See https://github.com/cloudsimplus/cloudsimplus/issues/590
/// @see GoogleMachineAndTaskEventsExample1
public class GoogleTaskEventsExample1 {
private static final String TASK_EVENTS_FILE = "workload/google-traces/task-events-sample-1.csv";
private static final String TASK_USAGE_FILE = "workload/google-traces/task-usage-sample-1.csv";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@
0,40,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493
10000,50,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493
10000,60,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493
20000,20,2,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.7,0.2493
30000,30,1,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493
50000,30,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,0.5,0.2493
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#Time stamp (microseconds),Machine ID,Event type (0-ADD 1-REMOVE 2-UPDATE),Platform ID,capacity:CPU (%),capacity:memory (%)
0,0,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,1
0,1,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,1
0,2,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,1
0,3,0,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,1
40000000,1,2,LGzk1Or/8Ildj2+Lqv0U82NLoni8+J/Yy0U=,1,0.75
Loading