first commit
This commit is contained in:
394
run.py
Executable file
394
run.py
Executable file
@ -0,0 +1,394 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
from typing import List, Tuple
|
||||
from yaml.scanner import ScannerError
|
||||
|
||||
import testcases
|
||||
|
||||
from implementations import IMPLEMENTATIONS
|
||||
from implementations import parse_filesize
|
||||
from interop import InteropRunner
|
||||
from testcases import MEASUREMENTS, TESTCASES
|
||||
|
||||
|
||||
def main():
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--debug",
|
||||
action="store_const",
|
||||
const=True,
|
||||
default=False,
|
||||
help="turn on debug logs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-m",
|
||||
"--manual-mode",
|
||||
action="store_true",
|
||||
help="only prepare the tests and print out the server and client run commands (to be executed manually)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
metavar="config.yml",
|
||||
help="File containing argument values"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-6", "--enable-ipv6", action="store_true", default=False, dest="v6", # dest allows accessing it
|
||||
help="Enables IPv6 execution in the interop runner"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s", "--server", help="server implementations (comma-separated)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c", "--client", help="client implementations (comma-separated)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--test",
|
||||
help="test cases (comma-separatated). Valid test cases are: "
|
||||
+ ", ".join([x.name() for x in TESTCASES + MEASUREMENTS]),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-l",
|
||||
"--log-dir",
|
||||
help="log directory",
|
||||
default="",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-f", "--save-files", help="save downloaded files if a test fails"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-i", "--implementation-directory",
|
||||
help="Directory containing the implementations."
|
||||
"This is prepended to the 'path' in the implementations.json file."
|
||||
"Default: .",
|
||||
default='.'
|
||||
)
|
||||
parser.add_argument(
|
||||
"-j", "--json", help="output the matrix to file in json format"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--venv-dir",
|
||||
help="dir to store venvs",
|
||||
default="",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--testbed",
|
||||
help="Runs the measurement in testbed mode. Requires a json file with client/server information"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bandwidth",
|
||||
help="Set a link bandwidth value which will be enforced using tc. Is only set in testbed mode on the remote hosts. Set values in tc syntax, e.g. 100mbit, 1gbit"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--delay",
|
||||
help="Add the chosen delay to packets sent to the client interface using tc. Set values in milliseconds, "
|
||||
"e.g. --delay 10ms"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reorder",
|
||||
nargs=2,
|
||||
help="Add random reordering to packets sent to the client interface using tc. It is required to set a "
|
||||
"delay value for using this option. Two percentage values are required for this option. The first is "
|
||||
"the percentage of packets immediately sent and the second is the correlation ,"
|
||||
"e.g. --reorder-packets 25%% 50%%"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--corruption",
|
||||
help="Add random noise corruption using tc. This option introduces a single bit error at a random offset "
|
||||
"in the packet. Set value in percentage, e.g. --corruption 0.1%%"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--loss",
|
||||
help="Add random packet loss specified in the 'tc' command in percentage, e.g. --loss 0.1%%"
|
||||
)
|
||||
# TODO: Maybe add option for packet duplication too
|
||||
|
||||
# Handle Pre-/Postscripts for server and client
|
||||
script_variable_help_msg = ("Available pos variables to use: "
|
||||
"interface (the interface we send/receive on, e.g enp123test), "
|
||||
"hostname, log_dir (the directory where all logs are saved to)")
|
||||
script_server_vars = ", www_dir (server root when serving files), certs_dir (folder of the certificates)"
|
||||
script_client_vars = ", sim_log_dir, download_dir"
|
||||
parser.add_argument(
|
||||
"-spre",
|
||||
"--server-prerunscript",
|
||||
default=[],
|
||||
nargs="*",
|
||||
metavar="SCRIPT",
|
||||
help="Add a bash script which should be executed before a test run on the server using pos. " + script_variable_help_msg + script_server_vars,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-sprehot",
|
||||
"--server-prerunscript-hot",
|
||||
default=[],
|
||||
nargs="*",
|
||||
metavar="SCRIPT",
|
||||
help="Add a bash script which should be executed at the first hold stage (client/server called begin()) " + script_variable_help_msg + script_server_vars,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-sposthot",
|
||||
"--server-postrunscript-hot",
|
||||
default=[],
|
||||
nargs="*",
|
||||
metavar="SCRIPT",
|
||||
help="Add a bash script which should be executed at the second hold stage (client/server called end()) " + script_variable_help_msg + script_server_vars,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-spost",
|
||||
"--server-postrunscript",
|
||||
default=[],
|
||||
nargs="*",
|
||||
metavar="SCRIPT",
|
||||
help="Add a bash script which should be executed after a test run on the server using pos. " + script_variable_help_msg + script_server_vars,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-cpre",
|
||||
"--client-prerunscript",
|
||||
default=[],
|
||||
nargs="*",
|
||||
metavar="SCRIPT",
|
||||
help="Add a bash script which should be executed before a test run on the client using pos. " + script_variable_help_msg + script_client_vars,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-cprehot",
|
||||
"--client-prerunscript-hot",
|
||||
default=[],
|
||||
nargs="*",
|
||||
metavar="SCRIPT",
|
||||
help="Add a bash script which should be executed at the first hold stage (client/server called begin()) " + script_variable_help_msg + script_client_vars,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-cposthot",
|
||||
"--client-postrunscript-hot",
|
||||
default=[],
|
||||
nargs="*",
|
||||
metavar="SCRIPT",
|
||||
help="Add a bash script which should be executed at the second hold stage (client/server called end()) " + script_variable_help_msg + script_client_vars,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-cpost",
|
||||
"--client-postrunscript",
|
||||
default=[],
|
||||
nargs="*",
|
||||
metavar="SCRIPT",
|
||||
help="Add a bash script which should be executed after a test run on the client using pos. " + script_variable_help_msg + script_client_vars,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--client-implementation-params",
|
||||
nargs="*",
|
||||
metavar="KEY=VALUE",
|
||||
help="",
|
||||
default=[]
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server-implementation-params",
|
||||
nargs="*",
|
||||
metavar="KEY=VALUE",
|
||||
help="",
|
||||
default=[]
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-server-aes-offload",
|
||||
action="store_const",
|
||||
const=True,
|
||||
default=False,
|
||||
help="turn server aes offload off",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-client-aes-offload",
|
||||
action="store_const",
|
||||
const=True,
|
||||
default=False,
|
||||
help="turn client aes offload off",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--filesize",
|
||||
help="Set the filesize of the transmitted file for all measurements. If no unit is specified MiB is assumed."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repetitions",
|
||||
metavar="N",
|
||||
type=int,
|
||||
help="Set the number of repetitions for all measurements."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--continue-on-error",
|
||||
action="store_true",
|
||||
help="Continue measurement even if a measurement fails."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-client-timestamps",
|
||||
action="store_true",
|
||||
help="Try to parse timestamps written by the client for computing goodput."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only-same-implementation",
|
||||
action="store_true",
|
||||
help="Test implementations only against their counterpart."
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.config:
|
||||
config_args = parse_config(args.config)
|
||||
parser.set_defaults(**config_args)
|
||||
# Ensure that every config file argument is defined in the parser
|
||||
for k, _ in config_args.items():
|
||||
if k not in args:
|
||||
sys.exit(f"Argument '{k}' from config file was not recognized by the parser.")
|
||||
args = parser.parse_args()
|
||||
|
||||
return args
|
||||
|
||||
def parse_config(config_file):
|
||||
try:
|
||||
with open(config_file, 'r') as f:
|
||||
model_config = yaml.safe_load(f)
|
||||
return model_config
|
||||
except ScannerError:
|
||||
sys.exit("config file syntax error!")
|
||||
except FileNotFoundError:
|
||||
sys.exit("config file not found!")
|
||||
|
||||
def get_dict_arg(arg):
|
||||
"""Given: list containing one KV pair
|
||||
per entry
|
||||
Return: dict containing all KV pairs
|
||||
from the list
|
||||
"""
|
||||
output = {}
|
||||
if not arg:
|
||||
return output
|
||||
for item in arg:
|
||||
if type(item) is dict:
|
||||
output = {**output, **item}
|
||||
else:
|
||||
try:
|
||||
k, v = item.split('=', 1)
|
||||
except ValueError:
|
||||
# handle entries without equals symbol as bool set to True
|
||||
output[item] = True
|
||||
continue
|
||||
try:
|
||||
output[k] = ast.literal_eval(v)
|
||||
except (ValueError, SyntaxError):
|
||||
output[k] = v
|
||||
return output
|
||||
|
||||
def get_impls(arg, availableImpls, role) -> List[str]:
|
||||
if not arg:
|
||||
return availableImpls
|
||||
impls = []
|
||||
arg = arg.replace(" ", "").replace("\n", "")
|
||||
for s in arg.replace(" ", "").split(","):
|
||||
if s not in availableImpls:
|
||||
sys.exit(role + " implementation " + s + " not found.")
|
||||
impls.append(s)
|
||||
return impls
|
||||
|
||||
def get_tests_and_measurements(
|
||||
arg,
|
||||
filesize,
|
||||
repetitions,
|
||||
) -> Tuple[List[testcases.TestCase], List[testcases.Measurement]]:
|
||||
if arg is None:
|
||||
return TESTCASES, MEASUREMENTS
|
||||
elif arg == "onlyTests":
|
||||
return TESTCASES, []
|
||||
elif arg == "onlyMeasurements":
|
||||
return [], MEASUREMENTS
|
||||
elif not arg:
|
||||
return [], []
|
||||
tests = []
|
||||
measurements = []
|
||||
for t in arg.split(","):
|
||||
if t in [tc.name() for tc in TESTCASES]:
|
||||
tests += [tc for tc in TESTCASES if tc.name() == t]
|
||||
elif t in [tc.name() for tc in MEASUREMENTS]:
|
||||
measurement = [tc for tc in MEASUREMENTS if tc.name() == t]
|
||||
if filesize:
|
||||
measurement[0].FILESIZE = parse_filesize(str(filesize), default_unit="MiB")
|
||||
if repetitions:
|
||||
measurement[0].REPETITIONS = int(repetitions)
|
||||
measurements += measurement
|
||||
else:
|
||||
print(
|
||||
(
|
||||
"Test case {} not found.\n"
|
||||
"Available testcases: {}\n"
|
||||
"Available measurements: {}"
|
||||
).format(
|
||||
t,
|
||||
", ".join([t.name() for t in TESTCASES]),
|
||||
", ".join([t.name() for t in MEASUREMENTS]),
|
||||
)
|
||||
)
|
||||
sys.exit()
|
||||
return tests, measurements
|
||||
|
||||
args = get_args()
|
||||
args.server_implementation_params = get_dict_arg(args.server_implementation_params)
|
||||
args.client_implementation_params = get_dict_arg(args.client_implementation_params)
|
||||
|
||||
tests, measurements = get_tests_and_measurements(
|
||||
args.test,
|
||||
args.filesize,
|
||||
args.repetitions
|
||||
)
|
||||
|
||||
# Check if reorder packets option is set without delay
|
||||
if args.reorder and (args.delay is None):
|
||||
print("--reorder requires --delay")
|
||||
return 1
|
||||
|
||||
if args.manual_mode and not args.testbed:
|
||||
print("Manual mode is currently only supported in testbed mode!")
|
||||
return 1
|
||||
|
||||
return InteropRunner(
|
||||
implementations=IMPLEMENTATIONS,
|
||||
implementations_directory=args.implementation_directory,
|
||||
servers=get_impls(args.server, IMPLEMENTATIONS, "Server"),
|
||||
clients=get_impls(args.client, IMPLEMENTATIONS, "Client"),
|
||||
tests=tests,
|
||||
measurements=measurements,
|
||||
output=args.json,
|
||||
debug=args.debug,
|
||||
manual_mode=args.manual_mode,
|
||||
log_dir=args.log_dir,
|
||||
save_files=args.save_files,
|
||||
venv_dir=args.venv_dir,
|
||||
testbed=args.testbed,
|
||||
bandwidth=args.bandwidth,
|
||||
server_pre_scripts=args.server_prerunscript,
|
||||
server_pre_hot_scripts=args.server_prerunscript_hot,
|
||||
server_post_hot_scripts=args.server_postrunscript_hot,
|
||||
server_post_scripts=args.server_postrunscript,
|
||||
client_pre_scripts=args.client_prerunscript,
|
||||
client_pre_hot_scripts=args.client_prerunscript_hot,
|
||||
client_post_hot_scripts=args.client_postrunscript_hot,
|
||||
client_post_scripts=args.client_postrunscript,
|
||||
reorder_packets=args.reorder,
|
||||
delay=args.delay,
|
||||
corruption=args.corruption,
|
||||
loss=args.loss,
|
||||
client_implementation_params=args.client_implementation_params,
|
||||
server_implementation_params=args.server_implementation_params,
|
||||
disable_server_aes_offload=args.disable_server_aes_offload,
|
||||
disable_client_aes_offload=args.disable_client_aes_offload,
|
||||
continue_on_error=args.continue_on_error,
|
||||
use_client_timestamps=args.use_client_timestamps,
|
||||
only_same_implementation=args.only_same_implementation,
|
||||
use_v6=args.v6,
|
||||
args=vars(args),
|
||||
).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user