395 lines
15 KiB
Python
Executable File
395 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import ast
|
|
import sys
|
|
import yaml
|
|
|
|
from typing import List, Tuple
|
|
from yaml.scanner import ScannerError
|
|
|
|
import testcases
|
|
|
|
from implementations import IMPLEMENTATIONS
|
|
from implementations import parse_filesize
|
|
from interop import InteropRunner
|
|
from testcases import MEASUREMENTS, TESTCASES
|
|
|
|
|
|
def main():
|
|
def get_args():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"-d",
|
|
"--debug",
|
|
action="store_const",
|
|
const=True,
|
|
default=False,
|
|
help="turn on debug logs",
|
|
)
|
|
parser.add_argument(
|
|
"-m",
|
|
"--manual-mode",
|
|
action="store_true",
|
|
help="only prepare the tests and print out the server and client run commands (to be executed manually)"
|
|
)
|
|
parser.add_argument(
|
|
"--config",
|
|
metavar="config.yml",
|
|
help="File containing argument values"
|
|
)
|
|
parser.add_argument(
|
|
"-6", "--enable-ipv6", action="store_true", default=False, dest="v6", # dest allows accessing it
|
|
help="Enables IPv6 execution in the interop runner"
|
|
)
|
|
parser.add_argument(
|
|
"-s", "--server", help="server implementations (comma-separated)"
|
|
)
|
|
parser.add_argument(
|
|
"-c", "--client", help="client implementations (comma-separated)"
|
|
)
|
|
parser.add_argument(
|
|
"-t",
|
|
"--test",
|
|
help="test cases (comma-separatated). Valid test cases are: "
|
|
+ ", ".join([x.name() for x in TESTCASES + MEASUREMENTS]),
|
|
)
|
|
parser.add_argument(
|
|
"-l",
|
|
"--log-dir",
|
|
help="log directory",
|
|
default="",
|
|
)
|
|
parser.add_argument(
|
|
"-f", "--save-files", help="save downloaded files if a test fails"
|
|
)
|
|
parser.add_argument(
|
|
"-i", "--implementation-directory",
|
|
help="Directory containing the implementations."
|
|
"This is prepended to the 'path' in the implementations.json file."
|
|
"Default: .",
|
|
default='.'
|
|
)
|
|
parser.add_argument(
|
|
"-j", "--json", help="output the matrix to file in json format"
|
|
)
|
|
parser.add_argument(
|
|
"--venv-dir",
|
|
help="dir to store venvs",
|
|
default="",
|
|
)
|
|
parser.add_argument(
|
|
"--testbed",
|
|
help="Runs the measurement in testbed mode. Requires a json file with client/server information"
|
|
)
|
|
parser.add_argument(
|
|
"--bandwidth",
|
|
help="Set a link bandwidth value which will be enforced using tc. Is only set in testbed mode on the remote hosts. Set values in tc syntax, e.g. 100mbit, 1gbit"
|
|
)
|
|
parser.add_argument(
|
|
"--delay",
|
|
help="Add the chosen delay to packets sent to the client interface using tc. Set values in milliseconds, "
|
|
"e.g. --delay 10ms"
|
|
)
|
|
parser.add_argument(
|
|
"--reorder",
|
|
nargs=2,
|
|
help="Add random reordering to packets sent to the client interface using tc. It is required to set a "
|
|
"delay value for using this option. Two percentage values are required for this option. The first is "
|
|
"the percentage of packets immediately sent and the second is the correlation ,"
|
|
"e.g. --reorder-packets 25%% 50%%"
|
|
)
|
|
parser.add_argument(
|
|
"--corruption",
|
|
help="Add random noise corruption using tc. This option introduces a single bit error at a random offset "
|
|
"in the packet. Set value in percentage, e.g. --corruption 0.1%%"
|
|
)
|
|
parser.add_argument(
|
|
"--loss",
|
|
help="Add random packet loss specified in the 'tc' command in percentage, e.g. --loss 0.1%%"
|
|
)
|
|
# TODO: Maybe add option for packet duplication too
|
|
|
|
# Handle Pre-/Postscripts for server and client
|
|
script_variable_help_msg = ("Available pos variables to use: "
|
|
"interface (the interface we send/receive on, e.g enp123test), "
|
|
"hostname, log_dir (the directory where all logs are saved to)")
|
|
script_server_vars = ", www_dir (server root when serving files), certs_dir (folder of the certificates)"
|
|
script_client_vars = ", sim_log_dir, download_dir"
|
|
parser.add_argument(
|
|
"-spre",
|
|
"--server-prerunscript",
|
|
default=[],
|
|
nargs="*",
|
|
metavar="SCRIPT",
|
|
help="Add a bash script which should be executed before a test run on the server using pos. " + script_variable_help_msg + script_server_vars,
|
|
)
|
|
parser.add_argument(
|
|
"-sprehot",
|
|
"--server-prerunscript-hot",
|
|
default=[],
|
|
nargs="*",
|
|
metavar="SCRIPT",
|
|
help="Add a bash script which should be executed at the first hold stage (client/server called begin()) " + script_variable_help_msg + script_server_vars,
|
|
)
|
|
parser.add_argument(
|
|
"-sposthot",
|
|
"--server-postrunscript-hot",
|
|
default=[],
|
|
nargs="*",
|
|
metavar="SCRIPT",
|
|
help="Add a bash script which should be executed at the second hold stage (client/server called end()) " + script_variable_help_msg + script_server_vars,
|
|
)
|
|
parser.add_argument(
|
|
"-spost",
|
|
"--server-postrunscript",
|
|
default=[],
|
|
nargs="*",
|
|
metavar="SCRIPT",
|
|
help="Add a bash script which should be executed after a test run on the server using pos. " + script_variable_help_msg + script_server_vars,
|
|
)
|
|
parser.add_argument(
|
|
"-cpre",
|
|
"--client-prerunscript",
|
|
default=[],
|
|
nargs="*",
|
|
metavar="SCRIPT",
|
|
help="Add a bash script which should be executed before a test run on the client using pos. " + script_variable_help_msg + script_client_vars,
|
|
)
|
|
parser.add_argument(
|
|
"-cprehot",
|
|
"--client-prerunscript-hot",
|
|
default=[],
|
|
nargs="*",
|
|
metavar="SCRIPT",
|
|
help="Add a bash script which should be executed at the first hold stage (client/server called begin()) " + script_variable_help_msg + script_client_vars,
|
|
)
|
|
parser.add_argument(
|
|
"-cposthot",
|
|
"--client-postrunscript-hot",
|
|
default=[],
|
|
nargs="*",
|
|
metavar="SCRIPT",
|
|
help="Add a bash script which should be executed at the second hold stage (client/server called end()) " + script_variable_help_msg + script_client_vars,
|
|
)
|
|
parser.add_argument(
|
|
"-cpost",
|
|
"--client-postrunscript",
|
|
default=[],
|
|
nargs="*",
|
|
metavar="SCRIPT",
|
|
help="Add a bash script which should be executed after a test run on the client using pos. " + script_variable_help_msg + script_client_vars,
|
|
)
|
|
parser.add_argument(
|
|
"--client-implementation-params",
|
|
nargs="*",
|
|
metavar="KEY=VALUE",
|
|
help="",
|
|
default=[]
|
|
)
|
|
parser.add_argument(
|
|
"--server-implementation-params",
|
|
nargs="*",
|
|
metavar="KEY=VALUE",
|
|
help="",
|
|
default=[]
|
|
)
|
|
parser.add_argument(
|
|
"--disable-server-aes-offload",
|
|
action="store_const",
|
|
const=True,
|
|
default=False,
|
|
help="turn server aes offload off",
|
|
)
|
|
parser.add_argument(
|
|
"--disable-client-aes-offload",
|
|
action="store_const",
|
|
const=True,
|
|
default=False,
|
|
help="turn client aes offload off",
|
|
)
|
|
parser.add_argument(
|
|
"--filesize",
|
|
help="Set the filesize of the transmitted file for all measurements. If no unit is specified MiB is assumed."
|
|
)
|
|
parser.add_argument(
|
|
"--repetitions",
|
|
metavar="N",
|
|
type=int,
|
|
help="Set the number of repetitions for all measurements."
|
|
)
|
|
parser.add_argument(
|
|
"--continue-on-error",
|
|
action="store_true",
|
|
help="Continue measurement even if a measurement fails."
|
|
)
|
|
parser.add_argument(
|
|
"--use-client-timestamps",
|
|
action="store_true",
|
|
help="Try to parse timestamps written by the client for computing goodput."
|
|
)
|
|
parser.add_argument(
|
|
"--only-same-implementation",
|
|
action="store_true",
|
|
help="Test implementations only against their counterpart."
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
if args.config:
|
|
config_args = parse_config(args.config)
|
|
parser.set_defaults(**config_args)
|
|
# Ensure that every config file argument is defined in the parser
|
|
for k, _ in config_args.items():
|
|
if k not in args:
|
|
sys.exit(f"Argument '{k}' from config file was not recognized by the parser.")
|
|
args = parser.parse_args()
|
|
|
|
return args
|
|
|
|
def parse_config(config_file):
|
|
try:
|
|
with open(config_file, 'r') as f:
|
|
model_config = yaml.safe_load(f)
|
|
return model_config
|
|
except ScannerError:
|
|
sys.exit("config file syntax error!")
|
|
except FileNotFoundError:
|
|
sys.exit("config file not found!")
|
|
|
|
def get_dict_arg(arg):
|
|
"""Given: list containing one KV pair
|
|
per entry
|
|
Return: dict containing all KV pairs
|
|
from the list
|
|
"""
|
|
output = {}
|
|
if not arg:
|
|
return output
|
|
for item in arg:
|
|
if type(item) is dict:
|
|
output = {**output, **item}
|
|
else:
|
|
try:
|
|
k, v = item.split('=', 1)
|
|
except ValueError:
|
|
# handle entries without equals symbol as bool set to True
|
|
output[item] = True
|
|
continue
|
|
try:
|
|
output[k] = ast.literal_eval(v)
|
|
except (ValueError, SyntaxError):
|
|
output[k] = v
|
|
return output
|
|
|
|
def get_impls(arg, availableImpls, role) -> List[str]:
|
|
if not arg:
|
|
return availableImpls
|
|
impls = []
|
|
arg = arg.replace(" ", "").replace("\n", "")
|
|
for s in arg.replace(" ", "").split(","):
|
|
if s not in availableImpls:
|
|
sys.exit(role + " implementation " + s + " not found.")
|
|
impls.append(s)
|
|
return impls
|
|
|
|
def get_tests_and_measurements(
|
|
arg,
|
|
filesize,
|
|
repetitions,
|
|
) -> Tuple[List[testcases.TestCase], List[testcases.Measurement]]:
|
|
if arg is None:
|
|
return TESTCASES, MEASUREMENTS
|
|
elif arg == "onlyTests":
|
|
return TESTCASES, []
|
|
elif arg == "onlyMeasurements":
|
|
return [], MEASUREMENTS
|
|
elif not arg:
|
|
return [], []
|
|
tests = []
|
|
measurements = []
|
|
for t in arg.split(","):
|
|
if t in [tc.name() for tc in TESTCASES]:
|
|
tests += [tc for tc in TESTCASES if tc.name() == t]
|
|
elif t in [tc.name() for tc in MEASUREMENTS]:
|
|
measurement = [tc for tc in MEASUREMENTS if tc.name() == t]
|
|
if filesize:
|
|
measurement[0].FILESIZE = parse_filesize(str(filesize), default_unit="MiB")
|
|
if repetitions:
|
|
measurement[0].REPETITIONS = int(repetitions)
|
|
measurements += measurement
|
|
else:
|
|
print(
|
|
(
|
|
"Test case {} not found.\n"
|
|
"Available testcases: {}\n"
|
|
"Available measurements: {}"
|
|
).format(
|
|
t,
|
|
", ".join([t.name() for t in TESTCASES]),
|
|
", ".join([t.name() for t in MEASUREMENTS]),
|
|
)
|
|
)
|
|
sys.exit()
|
|
return tests, measurements
|
|
|
|
args = get_args()
|
|
args.server_implementation_params = get_dict_arg(args.server_implementation_params)
|
|
args.client_implementation_params = get_dict_arg(args.client_implementation_params)
|
|
|
|
tests, measurements = get_tests_and_measurements(
|
|
args.test,
|
|
args.filesize,
|
|
args.repetitions
|
|
)
|
|
|
|
# Check if reorder packets option is set without delay
|
|
if args.reorder and (args.delay is None):
|
|
print("--reorder requires --delay")
|
|
return 1
|
|
|
|
if args.manual_mode and not args.testbed:
|
|
print("Manual mode is currently only supported in testbed mode!")
|
|
return 1
|
|
|
|
return InteropRunner(
|
|
implementations=IMPLEMENTATIONS,
|
|
implementations_directory=args.implementation_directory,
|
|
servers=get_impls(args.server, IMPLEMENTATIONS, "Server"),
|
|
clients=get_impls(args.client, IMPLEMENTATIONS, "Client"),
|
|
tests=tests,
|
|
measurements=measurements,
|
|
output=args.json,
|
|
debug=args.debug,
|
|
manual_mode=args.manual_mode,
|
|
log_dir=args.log_dir,
|
|
save_files=args.save_files,
|
|
venv_dir=args.venv_dir,
|
|
testbed=args.testbed,
|
|
bandwidth=args.bandwidth,
|
|
server_pre_scripts=args.server_prerunscript,
|
|
server_pre_hot_scripts=args.server_prerunscript_hot,
|
|
server_post_hot_scripts=args.server_postrunscript_hot,
|
|
server_post_scripts=args.server_postrunscript,
|
|
client_pre_scripts=args.client_prerunscript,
|
|
client_pre_hot_scripts=args.client_prerunscript_hot,
|
|
client_post_hot_scripts=args.client_postrunscript_hot,
|
|
client_post_scripts=args.client_postrunscript,
|
|
reorder_packets=args.reorder,
|
|
delay=args.delay,
|
|
corruption=args.corruption,
|
|
loss=args.loss,
|
|
client_implementation_params=args.client_implementation_params,
|
|
server_implementation_params=args.server_implementation_params,
|
|
disable_server_aes_offload=args.disable_server_aes_offload,
|
|
disable_client_aes_offload=args.disable_client_aes_offload,
|
|
continue_on_error=args.continue_on_error,
|
|
use_client_timestamps=args.use_client_timestamps,
|
|
only_same_implementation=args.only_same_implementation,
|
|
use_v6=args.v6,
|
|
args=vars(args),
|
|
).run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|