diff --git a/doc/FEATURES.md b/doc/FEATURES.md index 423cc116..4f4b9a7b 100644 --- a/doc/FEATURES.md +++ b/doc/FEATURES.md @@ -9,6 +9,12 @@ Turn internet-draft source into text and HTML. This supports XML files using [kramdown-rfc2629](https://github.com/cabo/kramdown-rfc2629) or [mmark](https://github.com/miekg/mmark) +```sh +$ make yanglint +``` + +Check YANG modules and examples for errors and warnings (see [YANG](YANG.md)). Also runs automatically during `make` if the `VALIDATE_YANG` environment variable is set. + ```sh $ make diff ``` diff --git a/doc/SETUP.md b/doc/SETUP.md index dc232293..d4470add 100644 --- a/doc/SETUP.md +++ b/doc/SETUP.md @@ -99,6 +99,32 @@ and find a directory that is on the path where you can install `mmark`. For these, I set `GOPATH=~/gocode`. +## pyang + +[`pyang`](https://github.com/mbj4668/pyang) is needed for markdown that uses `YANG-TREE ` to import an external yang file. + +```sh +$ pip install pyang +``` + + +## yanglint / libyang + +[`yanglint`](https://github.com/CESNET/libyang/tree/master/tools/lint) is part of the [`libyang`](https://github.com/CESNET/libyang) package. It's required only if you're validating YANG modules with `make yanglint` or with the `VALIDATE_YANG=1` environment variable during make. + +In late 2019, the libyang package is unfortunately not yet widely available as an rpm, deb, brew, etc. package in stable distros like for example the latest LTS, Bionic Beaver. So if you're using this feature, it may be necessary to install from source. This requires `cmake` and `libpcre3-dev`: + +```sh +git clone https://github.com/CESNET/libyang.git +mkdir libyang/build +pushd libyang/build +cmake -DCMAKE_INSTALL_PREFIX=/ .. +make +make install +popd +``` + + ## Other tools Some other helpful tools are listed in `config.mk`. diff --git a/doc/YANG.md b/doc/YANG.md new file mode 100644 index 00000000..5bbe3aad --- /dev/null +++ b/doc/YANG.md @@ -0,0 +1,49 @@ +# Working with YANG [RFC7950](https://tools.ietf.org/rfc7950) + +In the bad old days, YANG modules were hand-copied into markdown files, but +this became untenable as YANG started getting more usage. + +There are 3 YANG features that will be picked up during make, and which +will have enhanced error checking when the VALIDATE_YANG environment +variable is set to 1 or when running `make yanglint`: + + * `YANG-MODULE `: inserts the text of the .yang file + before building the markdown. Usually inside a diagram. Inserts + \ and \ tags in accordance with + [Section 3.2 of RFC 8407](https://tools.ietf.org/html/rfc8407#section-3.2) + + * when VALIDATE_YANG=1 or during yanglint, also performs a pyang lint check for IETF rules. + + * `YANG-TREE `: inserts the tree diagram of the .yang + file (as generated by pyang) before building the markdown, in + accordance with + [Section 3.4 of RFC 8407](https://tools.ietf.org/html/rfc8407#section-3.4) + + * `YANG-DATA `: inserts the text + of the .json file before building the markdown. + + * when VALIDATE_YANG=1 or when running `make yanglint`, also performs a yanglint check to ensure that + the example-data.json validates against the model in ietf-example.yang. + (yanglint is part of the [libyang](https://github.com/CESNET/libyang) + project) + +Note that the lint checks occur only for YANG modules and examples that +are imported with these commands from files external to the markdown. +If you have YANG examples or modules embedded in the markdown, they are +not examined by this feature. + +## Motivation + +It's useful to maintain separate YANG module files, because these can be +dropped directly into various tools that make use of them. By incorporating +the YANG module file into the I-D source markdown by reference instead of by +copy/paste, it becomes easier and more robust to ensure the actual .yang +files and the text of the draft don't accidentally diverge. + +Likewise, examples are incredibly useful for understanding how the yang +model is best used, and incredibly easy to diverge in the course of +refactoring the YANG model. + +It's hoped that by integrating support for external YANG files, the +quality of drafts that include YANG modules will improve with reduced +effort for authors. diff --git a/main.mk b/main.mk index 18d000cb..169a1934 100644 --- a/main.mk +++ b/main.mk @@ -10,6 +10,7 @@ include $(LIBDIR)/ghpages.mk include $(LIBDIR)/issues.mk include $(LIBDIR)/upload.mk include $(LIBDIR)/update.mk +include $(LIBDIR)/yanglint.mk ## Basic Targets .PHONY: txt html pdf @@ -32,16 +33,30 @@ REMOVE_LATEST = endif export XML_RESOURCE_ORG_PREFIX +# x.md.yangdeps files created by yang-inject.sh +-include $(addsuffix .yangdeps,$(wildcard *.md)) + %.xml: %.md @h=$$(head -1 $< | cut -c 1-4 -); set -o pipefail; \ if [ "$${h:0:1}" = $$'\ufeff' ]; then echo 'warning: BOM in $<' 1>&2; h="$${h:1:3}"; \ else h="$${h:0:3}"; fi; \ + if grep -q -E '^\s*YANG-(MODULE|DATA|TREE)' $< ; then \ + if [ "$(VALIDATE_YANG)" = "1" ]; then \ + if ! $(LIBDIR)/yang-check.sh $< ; then \ + echo "$(LIBDIR)/yang-check.sh $< failed" ; \ + exit 1 ; \ + fi ; \ + fi ;\ + if $(LIBDIR)/yang-inject.sh $< ; then \ + f="$<.withyang"; \ + else echo "$(LIBDIR)/yang-inject.sh $< failed" ; exit 1 ; fi ; \ + else f="$<" ; fi; \ if [ "$$h" = '---' ]; then \ - echo '$(subst ','"'"',cat $< $(MD_PREPROCESSOR) $(REMOVE_LATEST) | $(kramdown-rfc2629) > $@)'; \ - cat $< $(MD_PREPROCESSOR) $(REMOVE_LATEST) | $(kramdown-rfc2629) > $@; \ + echo '$(subst ','"'"',cat $$f $(MD_PREPROCESSOR) $(REMOVE_LATEST) | $(kramdown-rfc2629) > $@)'; \ + cat $$f $(MD_PREPROCESSOR) $(REMOVE_LATEST) | $(kramdown-rfc2629) > $@; \ elif [ "$$h" = '%%%' ]; then \ - echo '$(subst ','"'"',cat $< $(MD_PREPROCESSOR) $(REMOVE_LATEST) | $(mmark) -xml2 -page > $@)'; \ - cat $< $(MD_PREPROCESSOR) $(REMOVE_LATEST) | $(mmark) -xml2 -page > $@; \ + echo '$(subst ','"'"',cat $$f $(MD_PREPROCESSOR) $(REMOVE_LATEST) | $(mmark) -xml2 -page > $@)'; \ + cat $$f $(MD_PREPROCESSOR) $(REMOVE_LATEST) | $(mmark) -xml2 -page > $@; \ else \ ! echo "Unable to detect '%%%' or '---' in markdown file" 1>&2; \ fi @@ -183,6 +198,7 @@ COMMA := , clean:: -rm -f .tags $(targets_file) issues.json \ $(addsuffix .{txt$(COMMA)html$(COMMA)pdf},$(drafts)) index.html \ + $(addsuffix .{md.withyang$(COMMA)md.yangdeps},$(drafts)) \ $(addsuffix -[0-9][0-9].{xml$(COMMA)md$(COMMA)org$(COMMA)txt$(COMMA)html$(COMMA)pdf},$(drafts)) \ $(filter-out $(join $(drafts),$(draft_types)),$(addsuffix .xml,$(drafts))) \ $(uploads) $(draft_diffs) diff --git a/template/.gitignore b/template/.gitignore index 211017d8..b28c2b64 100644 --- a/template/.gitignore +++ b/template/.gitignore @@ -6,6 +6,10 @@ .tags *~ *.swp +.*.sw? +*.withyang +*.yangdeps +modules/ /*-[0-9][0-9].xml .refcache .targets.mk diff --git a/yang-check.sh b/yang-check.sh new file mode 100755 index 00000000..e133ae59 --- /dev/null +++ b/yang-check.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash + +input_md=$1 + +if [ ! -f "$input_md" ] ; then + echo "usage: $0 :" + echo " inserts yang into input.md+validates if VALIDATE_YANG=1" + echo " YANG-MODULE (pyang lint)" + echo " YANG-TREE (pyang lint)" + echo " YANG-DATA (yanglint validate json)" + echo " uses rsync to download imported modules to modules/ from iana" + echo "error: no input file: '$input_md'" + exit 1 +fi + +modules=$(grep YANG-MODULE $input_md | awk '{print $2;}' | tr "\n" " ") +tree_modules=$(grep YANG-TREE $input_md | awk '{print $2;}' | tr "\n" " ") +data_modules_str=$(grep YANG-DATA $input_md | awk '{print $2;}' | tr "\n" " ") + +data_json_str="$(grep YANG-DATA $input_md | awk '{print $3;}' | tr "\n" " ")" +all_modules="$(echo "$modules $tree_modules $data_modules_str" | tr " " "\n" | sort | uniq | tr "\n" " " | sed -e 's/^ *//' | sed -e 's/ *$//')" + +data_jsons=(); for d in $data_json_str; do data_jsons+=("$d"); done +data_modules=(); for d in $data_modules_str; do data_modules+=("$d"); done + +#echo "input=$input_md, data='${#data_jsons[@]}: $data_jsons', mods='$all_modules'" + +had_error=0 + +target_xml="$(echo ${input_md} | sed -e 's/.md$/.xml/')" +for ((i=0; i < ${#data_jsons[@]}; i++)); do + if [ ! -f "${data_jsons[i]}" ]; then + echo "no example data file ${data_jsons[i]}" + had_error=1 + fi +done +for mod in ${all_modules}; do + if [ ! -f ${mod} ]; then + echo "no module file ${mod}" + had_error=1 + fi +done + +if [ "${had_error}" != "0" ]; then + echo "error: some files not found, exiting" + exit 1 +fi + +fatal=0 +if ! which pyang 2>&1 > /dev/null ; then + echo "error in $0: pyang required; please install\n(pip install pyang)" + fatal=1 +fi + +if ! which yanglint 2>&1 > /dev/null ; then + echo "error in $0: yanglint required; please install libyang:" + echo "(install libyang-dev or equivalent if available, or build:)" + echo " git clone https://github.com/CESNET/libyang" + echo " mkdir libyang/build" + echo " pushd libyang/build" + echo " cmake .." + echo " make && sudo make install" + echo " popd" + echo " yanglint --help" + echo "if it can't load libyang.so, try something like:" + echo " cmake -DCMAKE_INSTALL_PREFIX=$$HOME/local-installs -DCMAKE_INSTALL_RPATH=$$HOME/local-installs/lib .." + fatal=1 +fi + +if [ "$fatal" != "0" ]; then + exit 1 +fi + +set -e + +if [ ! -d modules/ ]; then + mkdir -p modules/ +fi + +before=$(ls modules/ | wc -l) +after=${before} +last=$((before-1)) + +echo "checking for module dependencies (downloading into modules/...)" +while [ "${last}" != "${after}" ]; do + last=${after} + for mod in ${all_modules}; do + missing=$( ( pyang --verbose --path modules:. ${mod} 2>&1 || true ) | \ + grep "error: module" | \ + sed -e 's/.*: error: module "\([^"]*\)" not found in search path.*/\1/') + if [ "$missing" != "" ]; then + if ! which rsync 2>&1 > /dev/null ; then + echo "error in $0: rsync required; please install" + fi + + for mis in ${missing}; do + bash -e -x -c "rsync -cvz rsync.iana.org::assignments/yang-parameters/${mis}*.yang modules/" + done + fi + + # sample: + # err : Importing "ietf-routing-types" module into "ietf-dorms" failed. + + # unfortunately, you can't distinguish between missing and erroring, + # so in particular when multiple local modules import each other but + # the bottom one is missing things, skip pulling local stuff, and + # instead re-check those. TBD: set them up as dependencies? + + # TBD: maybe better to use yangcatalog.org to support I-D refs? + # see, e.g.: + # https://www.yangcatalog.org/yang-search/module_details.php?module=ietf-crypto-types@2019-07-02.yang + missing=$( ( yanglint -f json -D -V -p . -p modules ${mod} 2>&1 || true ) | \ + grep "err : Importing " | \ + sed -e 's/err : Importing "\([^"]*\)" module .* failed./\1/') + locals="" + while [ -f ${missing}.yang ]; do + already=0 + for loc in ${locals}; do + if [ "${loc}" = "${missing}.yang" ]; then + already=1 + fi + done + if [ "${already}" != "0" ]; then + break + fi + locals="${locals} ${missing}.yang" + missing=$( ( yanglint -f json -D -V -p . -p modules ${mod} 2>&1 || true ) | \ + grep "err : Importing " | \ + sed -e 's/err : Importing "\([^"]*\)" module .* failed./\1/') + done + if [ "$missing" != "" ]; then + for mis in ${missing}; do + if [ -f "$mis.yang" ]; then + missing=$( ( yanglint -f json -D -V -p . -p modules ${mis}.yang 2>&1 || true ) | \ + grep "err : Importing " | \ + sed -e 's/err : Importing "\([^"]*\)" module .* failed./\1/') + break + fi + done + + if ! which rsync 2>&1 > /dev/null ; then + echo "error in $0: rsync required; please install" + fi + for mis in ${missing}; do + bash -e -x -c "rsync -cvz rsync.iana.org::assignments/yang-parameters/${mis}*.yang modules/" + done + fi + done + after=$(ls modules/ | wc -l) +done + +echo "checking modules..." +for mod in ${all_modules}; do + if ! bash -x -e -c "pyang -E --verbose --ietf --lint --max-line-length 72 --path modules:. ${mod}" ; then + had_error=1 + fi +done + +echo "checking examples..." +for ((i=0; i < ${#data_modules[@]} && i < ${#data_jsons[@]}; i++)); do + # TBD: ideally, yanglint would have a "treat warnings as errors" mode, + # which ideally would be used here. + # TBD: ideally, also iana-if-type@2019-07-16.yang would not contain + # a redundant version that always reports this warning: + # warn: Module's revisions are not unique (2018-06-28). + bash -x -e -c "yanglint -f json -t data -D -s -V -p modules -p . -o /dev/null ${data_modules[i]} ${data_jsons[i]}" 2>&1 | grep -v "warn: Module's revisions are not unique (2018-06-28)" + if [ "${PIPESTATUS[0]}" != "0" ]; then + had_error=1 + fi +done + +if [ "${had_error}" != "0" ]; then + echo "encountered yang errors in ${input_md}" + exit 1 +fi +echo "yang passed checks in ${input_md}" + diff --git a/yang-fetch-imports.sh b/yang-fetch-imports.sh new file mode 100644 index 00000000..bd88c427 --- /dev/null +++ b/yang-fetch-imports.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +mod=$1 + +fatal=0 +if ! which pyang 2>&1 > /dev/null ; then + echo "error in $0: pyang required; please install\n(pip install pyang)" + fatal=1 +fi + +HAS_YANGLINT=1 +if ! which yanglint 2>&1 > /dev/null ; then + HAS_YANGLINT=0 +fi + +if [ "$fatal" != "0" ]; then + exit 1 +fi + +set -e + +if [ ! -d modules/ ]; then + mkdir -p modules/ +fi + +before=$(ls modules/ | wc -l) +after=${before} +last=$((before-1)) + +echo "checking for module dependencies (downloading into modules/...)" +while [ "${last}" != "${after}" ]; do + last=${after} + for mod in ${all_modules}; do + missing=$( ( pyang --verbose --path modules:. ${mod} 2>&1 || true ) | \ + grep "error: module" | \ + sed -e 's/.*: error: module "\([^"]*\)" not found in search path.*/\1/') + if [ "$missing" != "" ]; then + if ! which rsync 2>&1 > /dev/null ; then + echo "error in $0: rsync required; please install" + fi + + for mis in ${missing}; do + bash -e -x -c "rsync -cvz rsync.iana.org::assignments/yang-parameters/${mis}*.yang modules/" + done + fi + + # sample: + # err : Importing "ietf-routing-types" module into "ietf-dorms" failed. + + # unfortunately, you can't distinguish between missing and erroring, + # so in particular when multiple local modules import each other but + # the bottom one is missing things, skip pulling local stuff, and + # instead re-check those. TBD: set them up as dependencies? + + # TBD: maybe better to use yangcatalog.org to support I-D refs? + # see, e.g.: + # https://www.yangcatalog.org/yang-search/module_details.php?module=ietf-crypto-types@2019-07-02.yang + + # unfortunately, yanglint and pyang might use different dependencies. + # gotta check both. + if [ "${HAS_YANGLINT}" = "1" ]; then + missing=$( ( yanglint -f json -D -V -p . -p modules ${mod} 2>&1 || true ) | \ + grep "err : Importing " | \ + sed -e 's/err : Importing "\([^"]*\)" module .* failed./\1/') + locals="" + while [ -f ${missing}.yang ]; do + already=0 + for loc in ${locals}; do + if [ "${loc}" = "${missing}.yang" ]; then + already=1 + fi + done + if [ "${already}" != "0" ]; then + break + fi + locals="${locals} ${missing}.yang" + missing=$( ( yanglint -f json -D -V -p . -p modules ${mod} 2>&1 || true ) | \ + grep "err : Importing " | \ + sed -e 's/err : Importing "\([^"]*\)" module .* failed./\1/') + done + if [ "$missing" != "" ]; then + for mis in ${missing}; do + if [ -f "$mis.yang" ]; then + missing=$( ( yanglint -f json -D -V -p . -p modules ${mis}.yang 2>&1 || true ) | \ + grep "err : Importing " | \ + sed -e 's/err : Importing "\([^"]*\)" module .* failed./\1/') + break + fi + done + + if ! which rsync 2>&1 > /dev/null ; then + echo "error in $0: rsync required; please install" + fi + for mis in ${missing}; do + bash -e -x -c "rsync -cvz rsync.iana.org::assignments/yang-parameters/${mis}*.yang modules/" + done + fi + fi + done + after=$(ls modules/ | wc -l) +done + diff --git a/yang-inject.py b/yang-inject.py new file mode 100755 index 00000000..7368d0fa --- /dev/null +++ b/yang-inject.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import re +import datetime +import subprocess + +def main(args): + if len(args) != 2: + print('usage: %s ') + return -1 + + yang_re = re.compile(r'^ *YANG-(?P[A-Z]+) +(?P[^ ]+) *(?P[^ ]*)$') + + fname = args[1] + outfname = fname + '.withyang' + + with open(outfname, 'w') as outf: + line_no = 0 + for line in open(fname): + line_no += 1 + line = line.rstrip() + m = yang_re.match(line) + if not m: + print(line, file=outf) + else: + ytype = m.group('ytype') + if ytype == 'DATA': + in_data = m.group('in_ex') + if not in_data: + print('error: no input data file on line %d' % line_no) + return -4 + print(' inserting example json: %s' % (in_data)) + with open(in_data) as dataf: + print(dataf.read(), end='', file=outf) + elif ytype == 'MODULE': + in_data = m.group('in_module') + mod_name = in_data + if in_data.find('@') == -1: + if in_data.endswith('.yang'): + mod_name = in_data[:-5] + \ + datetime.datetime.now().strftime('@%Y-%m-%d') + \ + '.yang' + + print(' inserting module text from %s' % (in_data)) + print(' file %s' % mod_name, file=outf) + with open(in_data) as dataf: + print(dataf.read(), end='', file=outf) + print('', file=outf) + elif ytype == 'TREE': + in_data = m.group('in_module') + cmd = ['pyang', '--format', 'tree', '--tree-line-length', '69', '--path', 'modules:.', in_data] + print(' inserting pyang tree: %s' % ' '.join(cmd)) + psub = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = psub.communicate() + if psub.returncode != 0: + unexpected_errs = 0 + for errline in err.decode('utf-8').split('\n'): + errline = errline.strip() + # suppress and ignore module dependency errors and blanks + if not errline: + continue + if errline.find('not found in search path') != -1: + continue + unexpected_errs += 1 + print(errline, file=sys.stderr) + if unexpected_errs != 0: + raise subprocess.CalledProcessError(returncode=psub.returncode, cmd=cmd) + + for outline in out.decode('utf-8').split('\n'): + #print(outline) + print(outline, file=outf) + + return 0 + +if __name__=="__main__": + import sys + ret = main(sys.argv) + exit(ret) diff --git a/yang-inject.sh b/yang-inject.sh new file mode 100755 index 00000000..c378df80 --- /dev/null +++ b/yang-inject.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +input_md=$1 + +if [ ! -f "$input_md" ] ; then + echo "usage: $0 :" + echo " inserts yang into input.md" + echo " YANG-MODULE (inserts module)" + echo " YANG-TREE (builds tree view and inserts it)" + echo " YANG-DATA (inserts example.json)" + echo "error: no input file: '$input_md'" + exit 1 +fi + +modules=$(grep YANG-MODULE $input_md | awk '{print $2;}' | tr "\n" " ") +tree_modules=$(grep YANG-TREE $input_md | awk '{print $2;}' | tr "\n" " ") +data_modules_str=$(grep YANG-DATA $input_md | awk '{print $2;}' | tr "\n" " ") + +if [ "$modules" = "" -a "$data_modules_str" = "" -a "$tree_modules" = "" ]; then + if [ -f "$input_md.yangdeps" ]; then + echo "" > $input_md.yangdeps + fi + exit 0 +fi + +data_json_str="$(grep YANG-DATA $input_md | awk '{print $3;}' | tr "\n" " ")" +all_modules="$(echo "$modules $tree_modules $data_modules_str" | tr " " "\n" | sort | uniq | tr "\n" " " | sed -e 's/^ *//' | sed -e 's/ *$//')" + +data_jsons=(); for d in $data_json_str; do data_jsons+=("$d"); done +data_modules=(); for d in $data_modules_str; do data_modules+=("$d"); done + +#echo "input=$input_md, data='${#data_jsons[@]}: $data_jsons', mods='$all_modules'" + +had_error=0 + +target_xml="$(echo ${input_md} | sed -e 's/.md$/.xml/')" +for ((i=0; i < ${#data_jsons[@]}; i++)); do + if [ ! -f "${data_jsons[i]}" ]; then + echo "no example data file ${data_jsons[i]}" + had_error=1 + fi +done +for mod in ${all_modules}; do + if [ ! -f ${mod} ]; then + echo "no module file ${mod}" + had_error=1 + fi +done + +if [ "${had_error}" != "0" ]; then + echo "error: some files not found, exiting" + exit 1 +fi + +echo "${target_xml}: ${all_modules} ${data_json_str}" > ${input_md}.yangdeps + +fatal=0 +if ! which pyang 2>&1 > /dev/null ; then + echo "error in $0: pyang required; please install\n(pip install pyang)" + fatal=1 +fi + +if ! which python3 2>&1 > /dev/null ; then + echo "error in $0: python3 required; please install" + fatal=1 +fi + +if [ "$fatal" != "0" ]; then + exit 1 +fi + +echo "generating ${input_md}.withyang" +python3 lib/yang-inject.py ${input_md} || exit 1 + diff --git a/yanglint.mk b/yanglint.mk new file mode 100644 index 00000000..077eecbf --- /dev/null +++ b/yanglint.mk @@ -0,0 +1,16 @@ + +CHECK_TARGETS = $(addsuffix .checkyang,$(drafts_source)) +.PHONY: yanglint $(CHECK_TARGETS) + +$(CHECK_TARGETS): %.checkyang: % + @if grep -q -E '^\s*YANG-(MODULE|DATA|TREE)' $< ; then \ + if ! $(LIBDIR)/yang-check.sh $< ; then \ + echo "$(LIBDIR)/yang-check.sh $< failed" ; \ + exit 1 ; \ + fi ; \ + else \ + echo "(no yang imported in $<)" ; \ + fi + +yanglint: $(CHECK_TARGETS) +