diff options
Diffstat (limited to 'tests')
-rw-r--r-- | tests/Makefile.am | 7 | ||||
-rwxr-xr-x | tests/pdtest | 399 | ||||
-rw-r--r-- | tests/runtc.c | 482 |
3 files changed, 887 insertions, 1 deletions
diff --git a/tests/Makefile.am b/tests/Makefile.am index b1ac2bf..8086b9d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -19,7 +19,6 @@ ## if HAVE_CHECK - TESTS = check_main check_PROGRAMS = ${TESTS} @@ -37,5 +36,11 @@ check_main_CFLAGS = @check_CFLAGS@ check_main_LDADD = $(top_builddir)/libsigrokdecode.la @check_LIBS@ check_main_CPPFLAGS = $(CPPFLAGS_PYTHON) +endif +if HAVE_LIBSIGROK +bin_PROGRAMS = runtc +runtc_SOURCES = runtc.c +runtc_CPPFLAGS = $(CPPFLAGS_PYTHON) +runtc_LDFLAGS = -L/home/bert/sr/lib -lsigrok -lsigrokdecode endif diff --git a/tests/pdtest b/tests/pdtest new file mode 100755 index 0000000..59348d6 --- /dev/null +++ b/tests/pdtest @@ -0,0 +1,399 @@ +#!/usr/bin/env /usr/bin/python3 + +import os +import sys +from getopt import getopt +from tempfile import mkstemp +from subprocess import Popen, PIPE +from difflib import Differ + +DEBUG = False +VERBOSE = False + + +class E_syntax(Exception): + pass +class E_badline(Exception): + pass + +def INFO(msg, end='\n'): + if VERBOSE: + print(msg, end=end) + sys.stdout.flush() + + +def DBG(msg): + if DEBUG: + print(msg) + + +def ERR(msg): + print(msg, file=sys.stderr) + + +def usage(msg=None): + if msg: + print(msg.strip() + '\n') + print("""Usage: testpd [-dvarslR] [test, ...] + -d Turn on debugging + -v Verbose + -a All tests + -l List all tests + -s Show test(s) + -r Run test(s) + -R <directory> Save test reports to <directory> + <test> Protocol decoder name ("i2c") and optionally test name ("i2c/icc")""") + sys.exit() + + +def check_testcase(tc): + if 'pdlist' not in tc or not tc['pdlist']: + return("No protocol decoders") + if 'input' not in tc or not tc['input']: + return("No input") + if 'output' not in tc or not tc['output']: + return("No output") + for op in tc['output']: + if 'match' not in op: + return("No match in output") + + return None + + +def parse_testfile(path, pd, tc, op_type, op_class): + DBG("Opening '%s'" % path) + tclist = [] + for line in open(path).read().split('\n'): + try: + line = line.strip() + if len(line) == 0 or line[0] == "#": + continue + f = line.split() + if not tclist and f[0] != "test": + # That can't be good. + raise E_badline + key = f.pop(0) + if key == 'test': + if len(f) != 1: + raise E_syntax + # new testcase + tclist.append({ + 'pd': pd, + 'name': f[0], + 'pdlist': [], + 'output': [], + }) + elif key == 'protocol-decoder': + if len(f) < 1: + raise E_syntax + pd_spec = { + 'name': f.pop(0), + 'probes': [], + 'options': [], + } + while len(f): + if len(f) == 1: + # Always needs <key> <value> + raise E_syntax + a, b = f[:2] + f = f[2:] + if '=' not in b: + raise E_syntax + opt, val = b.split('=') + if a == 'probe': + try: + val = int(val) + except: + raise E_syntax + pd_spec['probes'].append([opt, val]) + elif a == 'option': + pd_spec['options'].append([opt, val]) + else: + raise E_syntax + tclist[-1]['pdlist'].append(pd_spec) + elif key == 'stack': + if len(f) < 2: + raise E_syntax + tclist[-1]['stack'] = f + elif key == 'input': + if len(f) != 1: + raise E_syntax + tclist[-1]['input'] = f[0] + elif key == 'output': + op_spec = { + 'pd': f.pop(0), + 'type': f.pop(0), + } + while len(f): + if len(f) == 1: + # Always needs <key> <value> + raise E_syntax + a, b = f[:2] + f = f[2:] + if a == 'class': + op_spec['class'] = b + elif a == 'match': + op_spec['match'] = b + else: + raise E_syntax + tclist[-1]['output'].append(op_spec) + else: + raise E_badline + except E_badline as e: + ERR("Invalid syntax in %s: line '%s'" % (path, line)) + return [] + except E_syntax as e: + ERR("Unable to parse %s: unknown line '%s'" % (path, line)) + return [] + + # If a specific testcase was requested, keep only that one. + if tc is not None: + target_tc = None + for t in tclist: + if t['name'] == tc: + target_tc = t + break + # ...and a specific output type + if op_type is not None: + target_oplist = [] + for op in target_tc['output']: + if op['type'] == op_type: + # ...and a specific output class + if op_class is None or ('class' in op and op['class'] == op_class): + target_oplist.append(op) + DBG("match on [%s]" % str(op)) + target_tc['output'] = target_oplist + if target_tc is None: + tclist = [] + else: + tclist = [target_tc] + for t in tclist: + error = check_testcase(t) + if error: + ERR("Error in %s: %s" % (path, error)) + return [] + + return tclist + + +def get_tests(testnames): + tests = [] + for testspec in testnames: + # Optional testspec in the form i2c/rtc + tc = op_type = op_class = None + ts = testspec.strip("/").split("/") + pd = ts.pop(0) + if ts: + tc = ts.pop(0) + if ts: + op_type = ts.pop(0) + if ts: + op_class = ts.pop(0) + path = os.path.join(decoders_dir, pd) + if not os.path.isdir(path): + # User specified non-existent PD + raise Exception("%s not found." % path) + path = os.path.join(decoders_dir, pd, "test/test.conf") + if not os.path.exists(path): + # PD doesn't have any tests yet + continue + tests.append(parse_testfile(path, pd, tc, op_type, op_class)) + + return tests + + +def diff_files(f1, f2): + t1 = open(f1).readlines() + t2 = open(f2).readlines() + diff = [] + d = Differ() + for line in d.compare(t1, t2): + if line[:2] in ('- ', '+ '): + diff.append(line.strip()) + + return diff + + +def run_tests(tests): + errors = 0 + results = [] + cmd = os.path.join(tests_dir, 'runtc') + for tclist in tests: + for tc in tclist: + args = [cmd] + for pd in tc['pdlist']: + args.extend(['-P', pd['name']]) + for label, probe in pd['probes']: + args.extend(['-p', "%s=%d" % (label, probe)]) + for option, value in pd['options']: + args.extend(['-o', "%s=%s" % (option, value)]) + args.extend(['-i', os.path.join(dumps_dir, tc['input'])]) + for op in tc['output']: + name = "%s/%s/%s" % (tc['pd'], tc['name'], op['type']) + opargs = ['-O', "%s:%s" % (op['pd'], op['type'])] + if 'class' in op: + opargs[-1] += ":%s" % op['class'] + name += "/%s" % op['class'] + if VERBOSE: + dots = '.' * (60 - len(name) - 2) + INFO("%s %s " % (name, dots), end='') + results.append({ + 'testcase': name, + }) + try: + fd, outfile = mkstemp() + os.close(fd) + opargs.extend(['-f', outfile]) + DBG("Running %s %s" % (cmd, ' '.join(args + opargs))) + stdout, stderr = Popen(args + opargs, stdout=PIPE, stderr=PIPE).communicate() + if stdout: + results[-1]['statistics'] = stdout.decode('utf-8').strip() + if stderr: + results[-1]['error'] = stderr.decode('utf-8').strip() + errors += 1 + match = "%s/%s/test/%s" % (decoders_dir, op['pd'], op['match']) + diff = diff_files(match, outfile) + if diff: + results[-1]['diff'] = diff + except Exception as e: + results[-1]['error'] = str(e) + finally: + os.unlink(outfile) + if VERBOSE: + if 'diff' in results[-1]: + INFO("Output mismatch") + elif 'error' in results[-1]: + error = results[-1]['error'] + if len(error) > 20: + error = error[:17] + '...' + INFO(error) + else: + INFO("OK") + gen_report(results[-1]) + + return results, errors + + +def gen_report(result): + out = [] + if 'error' in result: + out.append("Error:") + out.append(result['error']) + out.append('') + if 'diff' in result: + out.append("Test output mismatch:") + out.extend(result['diff']) + out.append('') + if 'statistics' in result: + out.extend(["Statistics:", result['statistics']]) + out.append('') + + if out: + text = "Testcase: %s\n" % result['testcase'] + text += '\n'.join(out) + else: + return + + if report_dir: + filename = result['testcase'].replace('/', '_') + open(os.path.join(report_dir, filename), 'w').write(text) + else: + print(text) + + +def show_tests(tests): + for tclist in tests: + for tc in tclist: + print("Testcase: %s/%s" % (tc['pd'], tc['name'])) + for pd in tc['pdlist']: + print(" Protocol decoder: %s" % pd['name']) + for label, probe in pd['probes']: + print(" Probe %s=%d" % (label, probe)) + for option, value in pd['options']: + print(" Option %s=%d" % (option, value)) + if 'stack' in tc: + print(" Stack: %s" % ' '.join(tc['stack'])) + print(" Input: %s" % tc['input']) + for op in tc['output']: + print(" Output:\n Protocol decoder: %s" % op['pd']) + print(" Type: %s" % op['type']) + if 'class' in op: + print(" Class: %s" % op['class']) + print(" Match: %s" % op['match']) + print() + + +def list_tests(tests): + for tclist in tests: + for tc in tclist: + for op in tc['output']: + line = "%s/%s/%s" % (tc['pd'], tc['name'], op['type']) + if 'class' in op: + line += "/%s" % op['class'] + print(line) + + +# +# main +# + +# project root +tests_dir = os.path.abspath(os.path.dirname(sys.argv[0])) +base_dir = os.path.abspath(os.path.join(os.curdir, tests_dir, os.path.pardir)) +dumps_dir = os.path.abspath(os.path.join(base_dir, os.path.pardir, 'sigrok-dumps')) +decoders_dir = os.path.abspath(os.path.join(base_dir, 'decoders')) + +if len(sys.argv) == 1: + usage() + +opt_all = opt_run = opt_show = opt_list = False +report_dir = None +opts, args = getopt(sys.argv[1:], "dvarslR:") +for opt, arg in opts: + if opt == '-d': + DEBUG = True + if opt == '-v': + VERBOSE = True + elif opt == '-a': + opt_all = True + elif opt == '-r': + opt_run = True + elif opt == '-s': + opt_show = True + elif opt == '-l': + opt_list = True + elif opt == '-R': + report_dir = arg + +if opt_run and opt_show: + usage("Use either -s or -r, not both.") +if args and opt_all: + usage("Specify either -a or tests, not both.") +if report_dir is not None and not os.path.isdir(report_dir): + usage("%s is not a directory" % report_dir) + +ret = 0 +try: + if args: + testlist = get_tests(args) + elif opt_all: + testlist = get_tests(os.listdir(decoders_dir)) + else: + usage("Specify either -a or tests.") + + if opt_run: + results, errors = run_tests(testlist) + ret = errors + elif opt_show: + show_tests(testlist) + elif opt_list: + list_tests(testlist) + else: + usage() +except Exception as e: + print("Error: %s" % str(e)) + if DEBUG: + raise + +sys.exit(ret) + diff --git a/tests/runtc.c b/tests/runtc.c new file mode 100644 index 0000000..78dbf5e --- /dev/null +++ b/tests/runtc.c @@ -0,0 +1,482 @@ +/* + * This file is part of the libsigrokdecode project. + * + * Copyright (C) 2013 Bert Vermeulen <bert@biot.com> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "../libsigrokdecode.h" +#include <libsigrok/libsigrok.h> +#include <stdlib.h> +#include <stdio.h> +#include <stdarg.h> +#include <unistd.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <time.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <glib.h> +#ifdef __LINUX__ +#include <sched.h> +#endif +#include "../config.h" + + +int debug = FALSE; +int statistics = FALSE; + +struct probe { + char *name; + int probe; +}; + +struct option { + char *key; + char *value; +}; + +struct pd { + char *name; + GSList *probes; + GSList *options; +}; + +struct output { + char *pd; + int type; + char *class; + int class_idx; + char *outfile; + int outfd; +}; + + +void logmsg(char *prefix, FILE *out, const char *format, va_list args) +{ + if (prefix) + fprintf(out, "%s", prefix); + vfprintf(out, format, args); + fprintf(out, "\n"); +} + +void DBG(const char *format, ...) +{ + va_list args; + + if (!debug) + return; + va_start(args, format); + logmsg("DBG: ", stdout, format, args); + va_end(args); +} + +void ERR(const char *format, ...) +{ + va_list args; + + va_start(args, format); + logmsg("Error: ", stderr, format, args); + va_end(args); +} + +int srd_log(void *cb_data, int loglevel, const char *format, va_list args) +{ + (void)cb_data; + + if (loglevel == SRD_LOG_ERR || loglevel == SRD_LOG_WARN) + logmsg("Error: srd: ", stderr, format, args); + else if (loglevel >= SRD_LOG_DBG && debug) + logmsg("DBG: srd: ", stdout, format, args); + + return SRD_OK; +} + +void usage(char *msg) +{ + if (msg) + fprintf(stderr, "%s\n", msg); + + //while((c = getopt(argc, argv, "dP:p:o:i:O:f:S")) != -1) { + printf("Usage: runtc [-dPpoiOf]\n"); + printf(" -d Debug\n"); + printf(" -P <protocol decoder>\n"); + printf(" -p <probename=probenum> (optional)\n"); + printf(" -o <probeoption=value> (optional)\n"); + printf(" -i <input file>\n"); + printf(" -O <output-pd:output-type[:output-class]>\n"); + printf(" -f <output file> (optional)\n"); + exit(msg ? 1 : 0); + +} + +static void srd_cb_ann(struct srd_proto_data *pdata, void *cb_data) +{ + struct srd_decoder *dec; + struct srd_proto_data_annotation *pda; + struct output *op; + GString *line; + int i; + char **dec_ann; + + DBG("Annotation from %s", pdata->pdo->di->inst_id); + op = cb_data; + pda = pdata->data; + dec = pdata->pdo->di->decoder; + + if (strcmp(pdata->pdo->di->inst_id, op->pd)) + /* This is not the PD selected for output. */ + return; + + if (op->class_idx != -1 && op->class_idx != pda->ann_format) + /* This output takes a specific annotation class, + * but not the one that just came in. */ + return; + + dec_ann = g_slist_nth_data(dec->annotations, pda->ann_format); + line = g_string_sized_new(256); + g_string_printf(line, "%"PRIu64"-%"PRIu64" %s: %s:", + pdata->start_sample, pdata->end_sample, + pdata->pdo->di->inst_id, dec_ann[0]); + for (i = 0; pda->ann_text[i]; i++) + g_string_append_printf(line, " \"%s\"", pda->ann_text[i]); + g_string_append(line, "\n"); + if (write(op->outfd, line->str, line->len) == -1) + ERR("Oops!"); + g_string_free(line, TRUE); + +} + +static void sr_cb(const struct sr_dev_inst *sdi, + const struct sr_datafeed_packet *packet, void *cb_data) +{ + const struct sr_datafeed_logic *logic; + struct srd_session *sess; + GVariant *gvar; + uint64_t samplerate; + int num_samples; + static int samplecnt = 0; + + sess = cb_data; + + switch (packet->type) { + case SR_DF_HEADER: + DBG("Received SR_DF_HEADER"); + if (sr_config_get(sdi->driver, sdi, NULL, SR_CONF_SAMPLERATE, + &gvar) != SR_OK) { + ERR("Getting samplerate failed"); + break; + } + samplerate = g_variant_get_uint64(gvar); + g_variant_unref(gvar); + if (srd_session_metadata_set(sess, SRD_CONF_SAMPLERATE, + g_variant_new_uint64(samplerate)) != SRD_OK) { + ERR("Setting samplerate failed"); + break; + } + if (srd_session_start(sess) != SRD_OK) { + ERR("Session start failed"); + break; + } + break; + case SR_DF_LOGIC: + logic = packet->payload; + num_samples = logic->length / logic->unitsize; + DBG("Received SR_DF_LOGIC: %d samples", num_samples); + srd_session_send(sess, samplecnt, samplecnt + num_samples, + logic->data, logic->length); + samplecnt += logic->length / logic->unitsize; + break; + case SR_DF_END: + DBG("Received SR_DF_END"); + break; + } + +} + +int get_stats(int stats[2]) +{ + FILE *f; + size_t len; + int tmp; + char *buf; + + stats[0] = stats[1] = -1; + if (!(f = fopen("/proc/self/status", "r"))) + return FALSE; + len = 128; + buf = malloc(len); + while (getline(&buf, &len, f) != -1) { + if (strcasestr(buf, "vmpeak:")) { + stats[0] = strtoul(buf + 10, NULL, 10); + } else if (strcasestr(buf, "vmsize:")) { + tmp = strtoul(buf + 10, NULL, 10); + if (tmp > stats[0]) + stats[0] = tmp; + } else if (strcasestr(buf, "vmhwm:")) { + stats[1] = strtoul(buf + 6, NULL, 10); + } else if (strcasestr(buf, "vmrss:")) { + tmp = strtoul(buf + 10, NULL, 10); + if (tmp > stats[0]) + stats[0] = tmp; + } + } + free(buf); + fclose(f); + + return TRUE; +} + +static int run_testcase(char *infile, GSList *pdlist, struct output *op) +{ + struct srd_session *sess; + struct srd_decoder *dec; + struct srd_decoder_inst *di, *prev_di; + struct pd *pd; + struct probe *probe; + struct option *option; + GVariant *gvar; + GHashTable *probes, *opts; + GSList *pdl, *l, *annl; + int idx; + char **dec_ann; + + if (op->outfile) { + if ((op->outfd = open(op->outfile, O_CREAT|O_WRONLY, 0600)) == -1) { + ERR("Unable to open %s for writing: %s", op->outfile, + strerror(errno)); + return FALSE; + } + } + + if (sr_session_load(infile) != SR_OK) + return FALSE; + + if (srd_session_new(&sess) != SRD_OK) + return FALSE; + sr_session_datafeed_callback_add(sr_cb, sess); + srd_pd_output_callback_add(sess, SRD_OUTPUT_ANN, srd_cb_ann, op); + + prev_di = NULL; + pd = NULL; + for (pdl = pdlist; pdl; pdl = pdl->next) { + pd = pdl->data; + if (srd_decoder_load(pd->name) != SRD_OK) + return FALSE; + + /* Instantiate decoder and pass in options. */ + opts = g_hash_table_new_full(g_str_hash, g_str_equal, NULL, + (GDestroyNotify)g_variant_unref); + for (l = pd->options; l; l = l->next) { + option = l->data; + g_hash_table_insert(opts, option->key, option->value); + } + if (!(di = srd_inst_new(sess, pd->name, opts))) + return FALSE; + g_hash_table_destroy(opts); + + /* Map probes. */ + if (pd->probes) { + probes = g_hash_table_new_full(g_str_hash, g_str_equal, NULL, + (GDestroyNotify)g_variant_unref); + for (l = pd->probes; l; l = l->next) { + probe = l->data; + gvar = g_variant_new_int32(probe->probe); + g_variant_ref_sink(gvar); + g_hash_table_insert(probes, probe->name, gvar); + } + if (srd_inst_probe_set_all(di, probes) != SRD_OK) + return FALSE; + g_hash_table_destroy(probes); + } + + + /* If this is not the first decoder in the list, stack it + * on top of the previous one. */ + if (prev_di) { + if (srd_inst_stack(sess, prev_di, di) != SRD_OK) { + ERR("Failed to stack decoder instances."); + return FALSE; + } + } + prev_di = di; + } + + /* Resolve top decoder's class index, so we can match. */ + dec = srd_decoder_get_by_id(pd->name); + if (op->class) { + if (op->type == SRD_OUTPUT_ANN) + annl = dec->annotations; + /* TODO can't dereference this for binary yet + else if (op->type == SRD_OUTPUT_BINARY) + annl = dec->binary; + */ + else + /* Only annotations and binary for now. */ + return FALSE; + idx = 0; + while(annl) { + dec_ann = annl->data; + /* TODO can't dereference this for binary yet */ + if (!strcmp(dec_ann[0], op->class)) { + op->class_idx = idx; + break; + } else + idx++; + annl = annl->next; + } + if (op->class_idx == -1) { + ERR("Output class '%s' not found in decoder %s.", + op->class, pd->name); + return FALSE; + } + } + + sr_session_start(); + sr_session_run(); + sr_session_stop(); + + srd_session_destroy(sess); + + if (op->outfile) + close(op->outfd); + + return TRUE; +} + +int main(int argc, char **argv) +{ + struct sr_context *ctx; + GSList *pdlist; + struct pd *pd; + struct probe *probe; + struct option *option; + struct output *op; + char c, *opt_infile, **kv, **opstr; + + op = malloc(sizeof(struct output)); + op->pd = NULL; + op->type = -1; + op->class = NULL; + op->class_idx = -1; + op->outfd = 1; + + pdlist = NULL; + opt_infile = NULL; + pd = NULL; + while((c = getopt(argc, argv, "dP:p:o:i:O:f:S")) != -1) { + switch(c) { + case 'd': + debug = TRUE; + break; + case 'P': + pd = g_malloc(sizeof(struct pd)); + pd->name = g_strdup(optarg); + pd->probes = pd->options = NULL; + pdlist = g_slist_append(pdlist, pd); + break; + case 'p': + case 'o': + if (g_slist_length(pdlist) == 0) { + /* No previous -P. */ + ERR("Syntax error at '%s'", optarg); + usage(NULL); + } + kv = g_strsplit(optarg, "=", 0); + if (!kv[0] || (!kv[1] || kv[2])) { + /* Need x=y. */ + ERR("Syntax error at '%s'", optarg); + g_strfreev(kv); + usage(NULL); + } + if (c == 'p') { + probe = malloc(sizeof(struct probe)); + probe->name = g_strdup(kv[0]); + probe->probe = strtoul(kv[1], 0, 10); + /* Apply to last PD. */ + pd->probes = g_slist_append(pd->probes, probe); + } else { + option = malloc(sizeof(struct option)); + option->key = g_strdup(kv[0]); + option->value = g_strdup(kv[1]); + /* Apply to last PD. */ + pd->options = g_slist_append(pd->options, option); + } + break; + case 'i': + opt_infile = optarg; + break; + case 'O': + opstr = g_strsplit(optarg, ":", 0); + if (!opstr[0] || !opstr[1]) { + /* Need at least abc:def. */ + ERR("Syntax error at '%s'", optarg); + g_strfreev(opstr); + usage(NULL); + } + op->pd = g_strdup(opstr[0]); + if (!strcmp(opstr[1], "annotation")) + op->type = SRD_OUTPUT_ANN; + else if (!strcmp(opstr[1], "binary")) + op->type = SRD_OUTPUT_BINARY; + else if (!strcmp(opstr[1], "python")) + op->type = SRD_OUTPUT_PYTHON; + else { + ERR("Unknown output type '%s'", opstr[1]); + g_strfreev(opstr); + usage(NULL); + } + if (opstr[2]) + op->class = g_strdup(opstr[2]); + g_strfreev(opstr); + break; + case 'f': + op->outfile = g_strdup(optarg); + op->outfd = -1; + break; + case 'S': + statistics = TRUE; + break; + default: + usage(NULL); + } + } + if (argc > optind) + usage(NULL); + if (g_slist_length(pdlist) == 0) + usage(NULL); + if (!opt_infile) + usage(NULL); + if (!op->pd || op->type == -1) + usage(NULL); + + if (sr_init(&ctx) != SR_OK) + return 1; + + srd_log_callback_set(srd_log, NULL); + if (srd_init(NULL) != SRD_OK) + return 1; + + run_testcase(opt_infile, pdlist, op); + + srd_exit(); + sr_exit(ctx); + + return 0; +} + + |