feat: re-run yang checks via celery (#7558)

* refactor: yang checks -> task

* chore: add periodic task

* chore: remove run_yang_model_checks.py

* test: add tests

* refactor: populate_yang_model_dirs -> task

* chore: remove populate_yang_model_dirs.py

* chore: remove python setup from bin/daily

bin/daily

@@ -5,7 +5,6 @@
 # This script is expected to be triggered by cron from
 # /etc/cron.d/datatracker
 export LANG=en_US.UTF-8
-export PYTHONIOENCODING=utf-8
 
 # Make sure we stop if something goes wrong:
 program=${0##*/}
@@ -17,10 +16,6 @@ cd $DTDIR/
 
 logger -p user.info -t cron "Running $DTDIR/bin/daily"
 
-# Set up the virtual environment
-source $DTDIR/env/bin/activate
-
-
 # Get IANA-registered yang models
 #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
 # Hardcode the rsync target to avoid any unwanted deletes:
@@ -30,9 +25,3 @@ rsync -avzq --delete /a/www/ietf-ftp/iana/yang-parameters/ /a/www/ietf-ftp/yang/
 # Get Yang models from Yangcatalog.
 #rsync -avzq rsync://rsync.yangcatalog.org:10873/yangdeps /a/www/ietf-ftp/yang/catalogmod/
 /a/www/ietf-datatracker/scripts/sync_to_yangcatalog
-
-# Populate the yang repositories
-$DTDIR/ietf/manage.py populate_yang_model_dirs -v0
-
-# Re-run yang checks on active documents
-$DTDIR/ietf/manage.py run_yang_model_checks -v0

ietf/submit/tasks.py

@@ -10,7 +10,8 @@
 
 from ietf.submit.models import Submission
 from ietf.submit.utils import (cancel_submission, create_submission_event, process_uploaded_submission,
-                               process_and_accept_uploaded_submission)
+                               process_and_accept_uploaded_submission, run_all_yang_model_checks,
+                               populate_yang_model_dirs)
 from ietf.utils import log
 
 
@@ -66,6 +67,12 @@ def cancel_stale_submissions():
         create_submission_event(None, subm, 'Submission canceled: expired without being posted')
 
 
+@shared_task
+def run_yang_model_checks_task():
+    populate_yang_model_dirs()
+    run_all_yang_model_checks()
+
+
 @shared_task(bind=True)
 def poke(self):
     log.log(f'Poked {self.name}, request id {self.request.id}')

ietf/submit/tests.py

@@ -49,6 +49,7 @@
 from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm
 from ietf.submit.models import Submission, Preapproval, SubmissionExtResource
 from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task
+from ietf.submit.utils import apply_yang_checker_to_draft, run_all_yang_model_checks
 from ietf.utils import tool_version
 from ietf.utils.accesstoken import generate_access_token
 from ietf.utils.mail import outbox, get_payload_text
@@ -3487,3 +3488,28 @@ def test_submission_checks(self):
                 "Your Internet-Draft failed at least one submission check.",
                 status_code=200,
             )
+
+
+class YangCheckerTests(TestCase):
+    @mock.patch("ietf.submit.utils.apply_yang_checker_to_draft")
+    def test_run_all_yang_model_checks(self, mock_apply):
+        active_drafts = WgDraftFactory.create_batch(3)
+        WgDraftFactory(states=[("draft", "expired")])
+        run_all_yang_model_checks()
+        self.assertEqual(mock_apply.call_count, 3)
+        self.assertCountEqual(
+            [args[0][1] for args in mock_apply.call_args_list],
+            active_drafts,
+        )
+
+    def test_apply_yang_checker_to_draft(self):
+        draft = WgDraftFactory()
+        submission = SubmissionFactory(name=draft.name, rev=draft.rev)
+        submission.checks.create(checker="my-checker")
+        checker = mock.Mock()
+        checker.name = "my-checker"
+        checker.symbol = "X"
+        checker.check_file_txt.return_value = (True, "whee", None, None, {})
+        apply_yang_checker_to_draft(checker, draft)
+        self.assertEqual(checker.check_file_txt.call_args, mock.call(draft.get_file_name()))
+

ietf/submit/utils.py

@@ -4,9 +4,11 @@
 
 import datetime
 import io
+import json
 import os
 import pathlib
 import re
+import sys
 import time
 import traceback
 import xml2rfc
@@ -15,6 +17,7 @@
 from shutil import move
 from typing import Optional, Union  # pyflakes:ignore
 from unidecode import unidecode
+from xym import xym
 
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -43,6 +46,7 @@
 from ietf.community.utils import update_name_contains_indexes_with_new_doc
 from ietf.submit.mail import ( announce_to_lists, announce_new_version, announce_to_authors,
     send_approval_request, send_submission_confirmation, announce_new_wg_00, send_manual_post_request )
+from ietf.submit.checkers import DraftYangChecker
 from ietf.submit.models import ( Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName,
     SubmissionCheck, SubmissionExtResource )
 from ietf.utils import log
@@ -1431,3 +1435,133 @@ def process_uploaded_submission(submission):
         submission.state_id = "uploaded"
         submission.save()
         create_submission_event(None, submission, desc="Completed submission validation checks")
+
+
+def apply_yang_checker_to_draft(checker, draft):
+    submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first()
+    if submission:
+        check = submission.checks.filter(checker=checker.name).order_by('-id').first()
+        if check:
+            result = checker.check_file_txt(draft.get_file_name())
+            passed, message, errors, warnings, items = result
+            items = json.loads(json.dumps(items))
+            new_res = (passed, errors, warnings, message)
+            old_res = (check.passed, check.errors, check.warnings, check.message) if check else ()
+            if new_res != old_res:
+                log.log(f"Saving new yang checker results for {draft.name}-{draft.rev}")
+                qs = submission.checks.filter(checker=checker.name).order_by('time')
+                submission.checks.filter(checker=checker.name).exclude(pk=qs.first().pk).delete()
+                submission.checks.create(submission=submission, checker=checker.name, passed=passed,
+                                         message=message, errors=errors, warnings=warnings, items=items,
+                                         symbol=checker.symbol)
+    else:
+        log.log(f"Could not run yang checker for {draft.name}-{draft.rev}: missing submission object")
+
+
+def run_all_yang_model_checks():
+    checker = DraftYangChecker()
+    for draft in Document.objects.filter(
+        type_id="draft",
+        states=State.objects.get(type="draft", slug="active"),
+    ):
+        apply_yang_checker_to_draft(checker, draft)
+
+
+def populate_yang_model_dirs():
+    """Update the yang model dirs
+
+     * All yang modules from published RFCs should be extracted and be
+       available in an rfc-yang repository.
+
+     * All valid yang modules from active, not replaced, Internet-Drafts
+       should be extracted and be available in a draft-valid-yang repository.
+
+     * All, valid and invalid, yang modules from active, not replaced,
+       Internet-Drafts should be available in a draft-all-yang repository.
+       (Actually, given precedence ordering, it would be enough to place
+       non-validating modules in a draft-invalid-yang repository instead).
+
+     * In all cases, example modules should be excluded.
+
+     * Precedence is established by the search order of the repository as
+       provided to pyang.
+
+     * As drafts expire, models should be removed in order to catch cases
+       where a module being worked on depends on one which has slipped out
+       of the work queue.
+
+    """
+    def extract_from(file, dir, strict=True):
+        saved_stdout = sys.stdout
+        saved_stderr = sys.stderr
+        xymerr = io.StringIO()
+        xymout = io.StringIO()
+        sys.stderr = xymerr
+        sys.stdout = xymout
+        model_list = []
+        try:
+            model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=-2)
+            for name in model_list:
+                modfile = moddir / name
+                mtime = file.stat().st_mtime
+                os.utime(str(modfile), (mtime, mtime))
+                if '"' in name:
+                    name = name.replace('"', '')
+                    modfile.rename(str(moddir / name))
+            model_list = [n.replace('"', '') for n in model_list]
+        except Exception as e:
+            log.log("Error when extracting from %s: %s" % (file, str(e)))
+        finally:
+            sys.stdout = saved_stdout
+            sys.stderr = saved_stderr
+        return model_list
+
+    # Extract from new RFCs
+
+    rfcdir = Path(settings.RFC_PATH)
+
+    moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR)
+    if not moddir.exists():
+        moddir.mkdir(parents=True)
+
+    latest = 0
+    for item in moddir.iterdir():
+        if item.stat().st_mtime > latest:
+            latest = item.stat().st_mtime
+
+    log.log(f"Extracting RFC Yang models to {moddir} ...")
+    for item in rfcdir.iterdir():
+        if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit():
+            if item.stat().st_mtime > latest:
+                model_list = extract_from(item, moddir)
+                for name in model_list:
+                    if not (name.startswith('ietf') or name.startswith('iana')):
+                        modfile = moddir / name
+                        modfile.unlink()
+
+    # Extract valid modules from drafts
+
+    six_months_ago = time.time() - 6 * 31 * 24 * 60 * 60
+
+    def active(dirent):
+        return dirent.stat().st_mtime > six_months_ago
+
+    draftdir = Path(settings.INTERNET_DRAFT_PATH)
+    moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
+    if not moddir.exists():
+        moddir.mkdir(parents=True)
+    log.log(f"Emptying {moddir} ...")
+    for item in moddir.iterdir():
+        item.unlink()
+
+    log.log(f"Extracting draft Yang models to {moddir} ...")
+    for item in draftdir.iterdir():
+        try:
+            if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
+                model_list = extract_from(item, moddir, strict=False)
+                for name in model_list:
+                    if name.startswith('example'):
+                        modfile = moddir / name
+                        modfile.unlink()
+        except UnicodeDecodeError as e:
+            log.log(f"Error processing {item.name}: {e}")

ietf/utils/management/commands/periodic_tasks.py

@@ -273,6 +273,16 @@ def create_default_tasks(self):
             ),
         )
 
+        PeriodicTask.objects.get_or_create(
+            name="Run Yang model checks",
+            task="ietf.submit.tasks.run_yang_model_checks_task",
+            defaults=dict(
+                enabled=False,
+                crontab=self.crontabs["daily"],
+                description="Re-run Yang model checks on all active drafts",
+            ),
+        )
+
     def show_tasks(self):
         for label, crontab in self.crontabs.items():
             tasks = PeriodicTask.objects.filter(crontab=crontab).order_by(