diff --git a/src/main/java/net/berack/upo/valpre/Main.java b/src/main/java/net/berack/upo/valpre/Main.java index 838b15b..47dd74c 100644 --- a/src/main/java/net/berack/upo/valpre/Main.java +++ b/src/main/java/net/berack/upo/valpre/Main.java @@ -20,7 +20,7 @@ public class Main { var param = Main.getParameters(program, subArgs); new SimulationBuilder(param.get("net")) .setCsv(param.get("csv")) - .setRuns(param.getOrDefault("runs", Integer::parseInt, 100)) + .setMaxRuns(param.getOrDefault("runs", Integer::parseInt, 100)) .setSeed(param.getOrDefault("seed", Long::parseLong, 2007539552L)) .setParallel(param.get("p") != null) .setEndCriteria(EndCriteria.parse(param.get("end"))) diff --git a/src/main/java/net/berack/upo/valpre/SimulationBuilder.java b/src/main/java/net/berack/upo/valpre/SimulationBuilder.java index eee9c90..b48bc15 100644 --- a/src/main/java/net/berack/upo/valpre/SimulationBuilder.java +++ b/src/main/java/net/berack/upo/valpre/SimulationBuilder.java @@ -2,13 +2,16 @@ package net.berack.upo.valpre; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.List; import java.util.concurrent.ExecutionException; import com.esotericsoftware.kryo.KryoException; +import net.berack.upo.valpre.sim.ConfidenceIndices; import net.berack.upo.valpre.sim.EndCriteria; import net.berack.upo.valpre.sim.Net; import net.berack.upo.valpre.sim.SimulationMultiple; import net.berack.upo.valpre.sim.stats.CsvResult; +import net.berack.upo.valpre.sim.stats.NodeStats; /** * This class is responsible for running the simulation. It parses the arguments @@ -21,6 +24,7 @@ public class SimulationBuilder { private boolean parallel; private Net net; private EndCriteria[] endCriteria; + private ConfidenceIndices confidences; /** * Create a new simulation for the given net. @@ -32,6 +36,7 @@ public class SimulationBuilder { try { var file = Parameters.getFileOrExample(netFile); this.net = Net.load(file); + this.confidences = new ConfidenceIndices(this.net); file.close(); } catch (FileNotFoundException e) { throw new IllegalArgumentException("Net file needed!"); @@ -50,16 +55,17 @@ public class SimulationBuilder { if (net == null) throw new IllegalArgumentException("Net needed!"); this.net = net; + this.confidences = new ConfidenceIndices(net); } /** - * Set the number of runs for the simulation. + * Set the maximum number of runs for the simulation. * * @param runs the number of runs * @throws IllegalArgumentException if the runs are less than 1 * @return this simulation */ - public SimulationBuilder setRuns(int runs) { + public SimulationBuilder setMaxRuns(int runs) { if (runs <= 0) throw new IllegalArgumentException("Runs must be greater than 0!"); @@ -117,16 +123,46 @@ public class SimulationBuilder { return this; } + /** + * Add a confidence index for the given node and stat. + * The confidence index is used to determine when the simulation should stop. + * + * + * @param node the node + * @param stat the stat to calculate the confidence index for + * @param confidence the confidence level expressed as a percentage [0,1] + * @param relError the relative error expressed as a percentage [0,1] + * @return this simulation + * @throws IllegalArgumentException if the node is invalid + * @throws IllegalArgumentException if the stat is invalid + * @throws IllegalArgumentException if the confidence is invalid + * @throws IllegalArgumentException if the relative error is invalid + */ + public SimulationBuilder addConfidenceIndex(String node, String stat, double confidence, double relError) { + if (!List.of(NodeStats.getOrderOfApply()).contains(stat)) + throw new IllegalArgumentException("Invalid statistic: " + stat); + if (confidence <= 0 || confidence > 1) + throw new IllegalArgumentException("Confidence must be between 0 and 1"); + if (relError <= 0 || relError > 1) + throw new IllegalArgumentException("Relative error must be between 0 and 1"); + + var index = this.net.getNodeIndex(node); + if (index < 0) + throw new IllegalArgumentException("Invalid node: " + node); + + this.confidences.add(index, stat, confidence, relError); + return this; + } + /** * Run the simulation with the given parameters. * At the end it prints the results and saves them to a CSV file if requested. * * @throws InterruptedException If the simulation is interrupted. - * @throws ExecutionException If the simulation fails. - * @throws KryoException If the simulation fails. - * @throws IOException If the simulation fails. + * @throws ExecutionException If the simulation has an error. + * @throws IOException If the CSV file has a problem. */ - public void run() throws InterruptedException, ExecutionException, KryoException, IOException { + public void run() throws InterruptedException, ExecutionException, IOException { var nano = System.nanoTime(); var sim = new SimulationMultiple(this.net); var summary = this.parallel diff --git a/src/main/java/net/berack/upo/valpre/sim/ConfidenceIndices.java b/src/main/java/net/berack/upo/valpre/sim/ConfidenceIndices.java new file mode 100644 index 0000000..e764b59 --- /dev/null +++ b/src/main/java/net/berack/upo/valpre/sim/ConfidenceIndices.java @@ -0,0 +1,132 @@ +package net.berack.upo.valpre.sim; + +import java.lang.reflect.Field; +import java.util.ArrayList; + +import net.berack.upo.valpre.sim.stats.NodeStats; +import net.berack.upo.valpre.sim.stats.Result; + +/** + * Confidence indices for a simulation. + * This class is used to store the confidence indices for a simulation. + * The confidence indices are used to determine when the simulation has + * reached a certain level of confidence. + */ +public class ConfidenceIndices { + private final String[] nodes; + private final NodeStats[] confidences; + private final NodeStats[] relativeErrors; + + /** + * Create a new confidence indices object for the given network. + * + * @param net the network to create the confidence indices for + */ + public ConfidenceIndices(Net net) { + var size = net.size(); + this.nodes = new String[size]; + this.confidences = new NodeStats[size]; + this.relativeErrors = new NodeStats[size]; + + for (var i = 0; i < size; i++) { + this.nodes[i] = net.getNode(i).name; + this.confidences[i] = new NodeStats(); + this.relativeErrors[i] = new NodeStats(); + this.relativeErrors[i].apply(_ -> 1.0); + } + } + + /** + * Add a confidence index to the simulation. The simulation will stop when the + * relative error of the confidence index is less than the given value. + * + * @param node The node to calculate the confidence index for. + * @param stat The statistic to calculate the confidence index for. + * @param confidence The confidence level of the confidence index. + * @param relError The relative error of the confidence index. + */ + public void add(int node, String stat, double confidence, double relError) { + if (node < 0 || node >= this.nodes.length) + throw new IllegalArgumentException("Invalid node: " + node); + + try { + Field field = NodeStats.class.getField(stat); + field.set(this.confidences[node], confidence); + field.set(this.relativeErrors[node], relError); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid statistic: " + stat); + } + } + + /** + * Calculate the relative errors of the statistics of the network. + * + * @param summary the summary of the network statistics + * @return the relative errors of the statistics + */ + public NodeStats[] calcRelativeErrors(Result.Summary summary) { + var errors = new NodeStats[this.nodes.length]; + for (var i = 0; i < this.confidences.length; i++) { + var node = this.nodes[i]; + var stat = summary.getSummaryOf(node); + + var confidence = this.confidences[i]; + var relativeError = stat.calcError(confidence); + relativeError.merge(stat.average, (err, avg) -> err / avg); + errors[i] = relativeError; + } + + return errors; + } + + /** + * Check if the errors are within the confidence indices. + * The errors within the confidence indices are calculated using the + * {@link #calcRelativeErrors(Result.Summary)} method. + * + * @param errors the relative errors of the statistics + * @return true if the simulation is ok, false otherwise + */ + public boolean isOk(NodeStats[] errors) { + for (var i = 0; i < this.relativeErrors.length; i++) { + var error = errors[i].clone(); + var relError = this.relativeErrors[i]; + + error.merge(relError, (err, rel) -> err - rel); + for (var value : error) + if (value > 0) + return false; + } + + return true; + } + + /** + * Get the errors of the statistics of the network. + * The errors are calculated using the + * {@link #calcRelativeErrors(Result.Summary)} method. + * Each error is formatted as a string in the format: "node:stat=value". + * + * @param errors the relative errors of the statistics + * @return the errors of the statistics + */ + public String[] getErrors(NodeStats[] errors) { + var statistics = NodeStats.getOrderOfApply(); + var retValues = new ArrayList(); + + for (var i = 0; i < this.relativeErrors.length; i++) { + var error = errors[i].clone(); + var relError = this.relativeErrors[i]; + error.merge(relError, (err, rel) -> err - rel); + + var j = 0; + for (var value : error) { + if (value > 0) + retValues.add("%s:%s=%0.3f".formatted(this.nodes[i], statistics[j], value)); + j += 1; + } + } + + return retValues.toArray(new String[0]); + } +} \ No newline at end of file diff --git a/src/main/java/net/berack/upo/valpre/sim/ServerNodeState.java b/src/main/java/net/berack/upo/valpre/sim/ServerNodeState.java index 30de8d5..2ada237 100644 --- a/src/main/java/net/berack/upo/valpre/sim/ServerNodeState.java +++ b/src/main/java/net/berack/upo/valpre/sim/ServerNodeState.java @@ -168,13 +168,11 @@ public class ServerNodeState { * otherwise */ public Event spawnArrivalToChild(double time, Rng rng) { - if (!this.children.isEmpty()) { - var random = rng.random(); - for (var child : this.children) { - random -= child.weight; - if (random <= 0) - return Event.newArrival(child.index, time); - } + var random = rng.random(); + for (var child : this.children) { + random -= child.weight; + if (random <= 0) + return Event.newArrival(child.index, time); } return null; } diff --git a/src/main/java/net/berack/upo/valpre/sim/SimulationMultiple.java b/src/main/java/net/berack/upo/valpre/sim/SimulationMultiple.java index 9210f50..2a7f483 100644 --- a/src/main/java/net/berack/upo/valpre/sim/SimulationMultiple.java +++ b/src/main/java/net/berack/upo/valpre/sim/SimulationMultiple.java @@ -87,4 +87,50 @@ public class SimulationMultiple { return results; } } + + /** + * Run the simulation multiple times with the given seed and end criteria. The + * simulation runs will stop when the relative error of the confidence index is + * less than the given value. + * The results are printed on the console. + * + * @param seed The seed to use for the random number generator. + * @param criterias The criteria to determine when to end the simulation. If + * null then the simulation will run until there are no more + * events. + * @return The statistics the network. + * @throws IllegalArgumentException If the confidence is not set. + */ + public void runIncremental(long seed, int runs, ConfidenceIndices confidences, EndCriteria... criterias) { + if (confidences == null) + throw new IllegalArgumentException("Confidence must be not null"); + + var rng = new Rng(seed); + var results = new Result.Summary(rng.getSeed()); + var output = new StringBuilder(); + var stop = false; + for (int i = 0; !stop; i++) { + var sim = new Simulation(this.net, rng, criterias); + var result = sim.run(); + results.add(result); + + if (i > 0) { + output.setLength(0); + output.append(String.format("\rSimulation [%6d]: ", i + 1)); + + var errors = confidences.calcRelativeErrors(results); + stop = confidences.isOk(errors); + + var errString = confidences.getErrors(errors); + var oneSting = String.join("], [", errString); + + output.append('[').append(oneSting).append("]"); + System.out.print(output); + } + } + + System.out.println("\nSimulation ended"); + System.out.println(results); + } + } diff --git a/src/main/java/net/berack/upo/valpre/sim/stats/CsvResult.java b/src/main/java/net/berack/upo/valpre/sim/stats/CsvResult.java index 549014e..1d99c99 100644 --- a/src/main/java/net/berack/upo/valpre/sim/stats/CsvResult.java +++ b/src/main/java/net/berack/upo/valpre/sim/stats/CsvResult.java @@ -44,7 +44,7 @@ public class CsvResult { try (var writer = new FileWriter(this.file)) { for (var result : results) { - for (var entry : result.nodes.entrySet()) { + for (var entry : result) { builder.append(result.seed).append(","); builder.append(entry.getKey()).append(","); builder.append(CsvResult.statsToCSV(entry.getValue())).append('\n'); diff --git a/src/main/java/net/berack/upo/valpre/sim/stats/NodeStats.java b/src/main/java/net/berack/upo/valpre/sim/stats/NodeStats.java index 76543be..5cdeb55 100644 --- a/src/main/java/net/berack/upo/valpre/sim/stats/NodeStats.java +++ b/src/main/java/net/berack/upo/valpre/sim/stats/NodeStats.java @@ -1,6 +1,7 @@ package net.berack.upo.valpre.sim.stats; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.function.BiFunction; import java.util.function.Function; @@ -14,7 +15,11 @@ import org.apache.commons.math3.distribution.TDistribution; * statistics are updated during simulation events, such as arrivals and * departures, and can be used to analyze the net's behavior and performance. */ -public class NodeStats implements Cloneable { +public class NodeStats implements Cloneable, Iterable { + private static final String[] ORDER_OF_APPLY = { "numArrivals", "numDepartures", "maxQueueLength", "avgQueueLength", + "avgWaitTime", "avgResponse", "busyTime", "waitTime", "unavailableTime", "responseTime", "lastEventTime", + "throughput", "utilization", "unavailable" }; + public double numArrivals = 0.0d; public double numDepartures = 0.0d; public double maxQueueLength = 0.0d; @@ -95,8 +100,6 @@ public class NodeStats implements Cloneable { /** * Apply a function to ALL the stats in this class. - * The only stats that are not updated with this function are the one that - * starts with max, min (since they are special) * The input of the function is the current value of the stat. * * @param func a function to apply @@ -106,19 +109,17 @@ public class NodeStats implements Cloneable { } /** - * A function used to merge tree stats. - * The only stats that are not updated with this function are the one that - * starts with max, min (since they are special) + * A function used to merge two sets of statistics. * - * @param other - * @param func + * @param other the other stats to merge + * @param func the function to merge the stats */ public NodeStats merge(NodeStats other, BiFunction func) { return NodeStats.operation(this, this, other, func); } @Override - protected NodeStats clone() { + public NodeStats clone() { try { return (NodeStats) super.clone(); } catch (CloneNotSupportedException e) { @@ -127,6 +128,23 @@ public class NodeStats implements Cloneable { } } + @Override + public Iterator iterator() { + return new Iterator<>() { + private int index = 0; + + @Override + public boolean hasNext() { + return this.index < ORDER_OF_APPLY.length; + } + + @Override + public Double next() { + return NodeStats.this.of(ORDER_OF_APPLY[this.index++]); + } + }; + } + /** * Get the value of the stat. * @@ -159,9 +177,7 @@ public class NodeStats implements Cloneable { * @return the order of the stats */ public static String[] getOrderOfApply() { - return new String[] { "numArrivals", "numDepartures", "avgQueueLength", "avgWaitTime", "avgResponse", - "busyTime", "waitTime", "unavailableTime", "responseTime", "lastEventTime", "throughput", "utilization", - "unavailable" }; + return ORDER_OF_APPLY; } /** @@ -185,7 +201,7 @@ public class NodeStats implements Cloneable { BiFunction func) { save.numArrivals = func.apply(val1.numArrivals, val2.numArrivals); save.numDepartures = func.apply(val1.numDepartures, val2.numDepartures); - // save.maxQueueLength = func.apply(val1.maxQueueLength, val2.maxQueueLength); + save.maxQueueLength = func.apply(val1.maxQueueLength, val2.maxQueueLength); save.avgQueueLength = func.apply(val1.avgQueueLength, val2.avgQueueLength); save.avgWaitTime = func.apply(val1.avgWaitTime, val2.avgWaitTime); save.avgResponse = func.apply(val1.avgResponse, val2.avgResponse); @@ -264,11 +280,27 @@ public class NodeStats implements Cloneable { * @return the error of the values */ public NodeStats calcError(double alpha) { + return this.calcError(new NodeStats().apply(_ -> alpha)); + } + + /** + * Calculates the error at the selected alpha level for each NodeStats. + * This method computes the error for the average and standard deviation values, + * considering the sample size and the confidence level (alpha). + * The result is adjusted using a t-distribution to account for the variability + * in smaller sample sizes. + * + * @param distribution the t-distribution to use + * @param stdDev the standard deviation of the values + * @param alpha the alpha values for each statistics + * @return the error of the values + */ + public NodeStats calcError(NodeStats alpha) { var n = this.stats.size(); var distr = new TDistribution(null, n - 1); - var tValue = distr.inverseCumulativeProbability(alpha); + var tValue = alpha.clone().apply(a -> distr.inverseCumulativeProbability(a)); - return this.stdDev().apply(std -> tValue * (std / Math.sqrt(n))); + return this.stdDev().merge(tValue, (std, t) -> t * (std / Math.sqrt(n))); } /** diff --git a/src/main/java/net/berack/upo/valpre/sim/stats/Result.java b/src/main/java/net/berack/upo/valpre/sim/stats/Result.java index da07937..3790739 100644 --- a/src/main/java/net/berack/upo/valpre/sim/stats/Result.java +++ b/src/main/java/net/berack/upo/valpre/sim/stats/Result.java @@ -5,6 +5,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; /** * Represents the statistics of a network simulation. @@ -12,7 +13,7 @@ import java.util.Map; * nodes, including the number of arrivals and departures, the maximum queue * length, the busy time, and the response time. */ -public class Result { +public class Result implements Iterable> { public final Map nodes; public final long seed; public final double simulationTime; @@ -39,6 +40,11 @@ public class Result { return buildPrintable(this.seed, this.simulationTime, this.timeElapsedMS, this.nodes); } + @Override + public java.util.Iterator> iterator() { + return this.nodes.entrySet().iterator(); + } + private static String buildPrintable(long seed, double simTime, double timeMS, Map nodes) { var size = (int) Math.ceil(Math.max(Math.log10(simTime), 1)); var iFormat = "%" + size + ".0f";