ci: Add n310 rf tests to monopipeline

This changes adds in n310 rf test support that was previously in
buildbot.

Signed-off-by: Steven Koo <steven.koo@ni.com>

.ci/templates/job-uhd-hardware-tests.yml

@@ -1,106 +0,0 @@
-parameters:
-- name: dutMatrix
-  type: object
-- name: runDevTest
-  type: boolean
-  default: true
-- name: runPyTest
-  type: boolean
-  default: true
-
-jobs:
-- job: build_uhd_run_hardware_test
-  displayName: Build uhd and run hardware test
-  strategy:
-    matrix: ${{ parameters.dutMatrix }}
-
-  pool:
-    name: 'de-dre-lab'
-    demands: uhd_ats -equals $(pipelineAgent)
-
-  steps:
-  - checkout: ettus-rts
-    clean: true
-
-  - checkout: self
-    clean: true
-
-  - template: steps-build-uhd-make.yml
-    parameters:
-      uhdSrcDir: $(Build.SourcesDirectory)/uhddev
-      uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build
-
-  - script: |
-      mkdir -p $(Common.TestResultsDirectory)/devtest
-      cd $(Common.TestResultsDirectory)/devtest
-      python3 $(Build.SourcesDirectory)/uhddev/host/tests/devtest/run_testsuite.py \
-        --src-dir $(Build.SourcesDirectory)/uhddev/host/tests/devtest \
-        --devtest-pattern $(devtestPattern) --args addr=$(devAddr),type=$(devType) \
-        --build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \
-        --python-interp python3 --xml
-    continueOnError: true
-    condition: and(succeeded(), eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'), eq(variables['devBus'], 'ip'))
-    displayName: Run devtest for IP
-
-  - script: |
-      mkdir -p $(Common.TestResultsDirectory)/devtest
-      cd $(Common.TestResultsDirectory)/devtest
-      python3 $(Build.SourcesDirectory)/uhddev/host/tests/devtest/run_testsuite.py \
-        --src-dir $(Build.SourcesDirectory)/uhddev/host/tests/devtest \
-        --devtest-pattern $(devtestPattern) --args serial=$(devSerial),type=$(devType) \
-        --build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \
-        --python-interp python3 --xml
-    continueOnError: true
-    condition: and(succeeded(), eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'), ne(variables['devBus'], 'ip'))
-    displayName: Run devtest for PCIe / USB
-
-  - script: |
-      mkdir -p $(Common.TestResultsDirectory)/pytest
-      cd $(Build.SourcesDirectory)/ettus-rts/config/remote/python_tests
-
-      export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
-      export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
-      export PYTHONPATH=$(Build.BinariesDirectory)/uhddev/build/python/build/lib:$PYTHONPATH
-
-      python3 automated_main.py --ats_config $(pytestAtsConfig) --dut $(pytestDUT) \
-        --results_path $(Common.TestResultsDirectory)/pytest
-    continueOnError: true
-    condition: and(succeeded(), eq('${{ parameters.runPyTest }}', true), eq(variables['deviceEnabled'], 'true'))
-    displayName: Run pytests
-
-  - script: |
-      cd $(Common.TestResultsDirectory)/devtest
-      python3 $(Build.SourcesDirectory)/uhddev/.ci/utils/format_devtest_junitxml.py \
-        $(Common.TestResultsDirectory)/devtest \
-        $(Common.TestResultsDirectory)/devtest/devtestresults.xml
-    continueOnError: true
-    condition: and(succeeded(), eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'))
-    displayName: Format devtest xml
-
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFormat: 'JUnit'
-      testResultsFiles: '$(Common.TestResultsDirectory)/devtest/devtestresults.xml'
-      testRunTitle: $(devName) devtest
-      buildConfiguration: 'Release'
-      mergeTestResults: true
-      failTaskOnFailedTests: true
-    condition: and(eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'))
-    displayName: Upload devtest results
-
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFormat: 'JUnit'
-      testResultsFiles: '$(Common.TestResultsDirectory)/pytest/**/*.xml'
-      testRunTitle: $(devName) pytest
-      buildConfiguration: 'Release'
-      mergeTestResults: true
-    condition: and(succeeded(), eq('${{ parameters.runPyTest }}', true), eq(variables['deviceEnabled'], 'true'))
-    displayName: Upload pytest results
-
-  - script: |
-      echo "A previous step failed. See steps with warnings."
-      echo "Marking the overall build status as a failure."
-      exit 1
-    condition: in(variables['Agent.JobStatus'],  'SucceededWithIssues')
-    displayName: Mark build as failure

.ci/templates/job-uhd-rf-tests-pebbles.yml

@@ -0,0 +1,40 @@
+parameters:
+- name: testOS
+  type: string
+  values:
+    - ubuntu1804
+- name: uhdSrcDir
+  type: string
+- name: uhdArtifactSource
+  type: string
+  default: current
+- name: testDevices
+  type: string
+  default: 'n3xx'
+
+jobs:
+- template: job-uhd-rf-tests.yml
+  parameters:
+    suiteName: 'pebbles'
+    testOS: '${{ parameters.testOS }}'
+    toolset: 'make'
+    uhdSrcDir: '${{ parameters.uhdSrcDir }}'
+    uhdArtifactSource: '${{ parameters.uhdArtifactSource }}'
+    redisHost: 'ettus-pebbles'
+    dutMatrix:
+      ${{ if contains(parameters.testDevices, 'n3xx') }}:
+        pebbles-n310-0:
+          devType: 'n3xx'
+          devModel: 'n310'
+          devName: pebbles-n310-0
+          devSerial: '3176DDF'
+          devHostname: 'ni-n3xx-3176DDF'
+          devBus: 'ip'
+          devAddr: '192.168.40.17'
+          sfpAddrs: '192.168.10.17,192.168.40.17'
+          devFpga: 'HG'
+          devtestPattern: 'n3x0'
+          devSDImage: gnuradio-image-ni-sulfur-rev11-mender.sdimg.bz2
+          devLabgridConfig: .ci/templates/tests/pebbles-labgrid/device-configs/pebbles-n310-0.yml
+          pytestDUT: 'n310-0'
+          pytestAtsConfig: 'uhd_oss_ats'

.ci/templates/job-uhd-rf-tests.yml

@@ -0,0 +1,164 @@
+parameters:
+- name: suiteName
+  type: string
+- name: testOS
+  type: string
+  values:
+    - ubuntu1804
+- name: toolset
+  type: string
+  values:
+    - make
+- name: uhdSrcDir
+  type: string
+- name: uhdArtifactSource
+  type: string
+- name: redisHost
+  type: string
+- name: dutMatrix
+  type: object
+
+jobs:
+- job: build_uhd_run_hardware_test
+  displayName: Build uhd and run hardware test
+  timeoutInMinutes: 90
+  strategy:
+    matrix: ${{ parameters.dutMatrix }}
+  pool:
+    name: 'de-dre-lab'
+    demands:
+    - suiteName -equals ${{ parameters.suiteName }}
+    - testOS -equals ${{ parameters.testOS }}
+  variables:
+  - group: sdr-pipeline-vars
+  workspace:
+    clean: outputs
+
+  steps:
+  - checkout: ettus-rts
+    clean: true
+
+  - checkout: self
+    clean: true
+
+  - script: |
+      if [ "${{ parameters.uhdArtifactSource }}" = "current" ]; then
+        echo '##vso[task.setvariable variable=uhd_artifact_directory]'$(Agent.BuildDirectory)
+      else
+        echo '##vso[task.setvariable variable=uhd_artifact_directory]'$(Agent.BuildDirectory)/${{ parameters.uhdArtifactSource }}
+      fi
+    displayName: Set uhddev pipeline artifact location
+
+  - download: ${{ parameters.uhdArtifactSource }}
+    artifact: uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}
+    displayName: Download uhddev-${{ parameters.testOS }}-${{ parameters.toolset }} artifact
+
+  - download: ${{ parameters.uhdArtifactSource }}
+    artifact: uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}
+    displayName: Download uhddev-${{ parameters.testOS }}-${{ parameters.toolset }} artifact
+
+  - task: ExtractFiles@1
+    inputs:
+      archiveFilePatterns: $(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
+      destinationFolder: $(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}
+      cleanDestinationFolder: false
+      overwriteExistingFiles: true
+    displayName: Extract uhddev-${{ parameters.testOS }}-${{ parameters.toolset }} artifact
+
+  - download: ${{ parameters.uhdArtifactSource }}
+    artifact: gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}
+    displayName: Download gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }} artifact
+
+  - task: ExtractFiles@1
+    inputs:
+      archiveFilePatterns: $(uhd_artifact_directory)/gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}/gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
+      destinationFolder: $(uhd_artifact_directory)/gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}
+      cleanDestinationFolder: false
+      overwriteExistingFiles: true
+    displayName: Extract gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }} artifact
+
+  - download: ${{ parameters.uhdArtifactSource }}
+    artifact: $(devType)-images
+    # Only sync the bz2 sdimg since the bmap
+    # is incompatible with mender
+    patterns: |
+      **/*.bz2
+      fitImage-manufacturing
+    displayName: Download $(devType)-images artifact
+    condition: and(succeeded(), eq(variables.devType, 'n3xx'))
+
+  - script: |
+      mkdir -p $(Build.BinariesDirectory)/uhddev/build/fpga_images
+      cd $(Build.BinariesDirectory)/uhddev/build
+      rm -rf fpga_images/*
+      python3 $(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev/build/utils/uhd_images_downloader.py -t $(devModel) -i fpga_images \
+        -b $(sdr-fileserver)
+      if [ "$(devType)" = "b200" ]; then
+        python3 $(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev/utils/uhd_images_downloader.py -t b2xx_common -i fpga_images \
+          -b $(sdr-fileserver)
+      fi
+    displayName: Download FPGA Images
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/pytest
+      cd $(Common.TestResultsDirectory)/pytest
+
+      export UHD_INSTALL_PATH=$(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev/build-installed
+      export GNURADIO_GRETTUS_INSTALL_PATH=$(uhd_artifact_directory)/gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}/gnuradio/build-installed
+      export PATH=$UHD_INSTALL_PATH/bin:$PATH
+      export PATH=$GNURADIO_GRETTUS_INSTALL_PATH/bin:$PATH
+      export PYTHON_VERSION=$(python3 -c "import sys; print('python'+str(sys.version_info.major) + '.' + str(sys.version_info.minor))")
+      export UHD_IMAGES_DIR=$(Build.BinariesDirectory)/uhddev/build/fpga_images
+
+      if [ -d "$UHD_INSTALL_PATH/lib64" ]; then
+        export PATH=$UHD_INSTALL_PATH/lib64/uhd/examples:$PATH
+        export LD_LIBRARY_PATH=$UHD_INSTALL_PATH/lib64:$LD_LIBRARY_PATH
+        export LD_LIBRARY_PATH=$GNURADIO_GRETTUS_INSTALL_PATH/lib64:$LD_LIBRARY_PATH
+        export PYTHONPATH=$UHD_INSTALL_PATH/lib64/$PYTHON_VERSION/site-packages:$PYTHONPATH
+        export PYTHONPATH=$GNURADIO_GRETTUS_INSTALL_PATH/lib64/python3/dist-packages:$PYTHONPATH
+      else
+        export PATH=$UHD_INSTALL_PATH/lib/uhd/examples:$PATH
+        export LD_LIBRARY_PATH=$UHD_INSTALL_PATH/lib:$LD_LIBRARY_PATH
+        export LD_LIBRARY_PATH=$GNURADIO_GRETTUS_INSTALL_PATH/lib:$LD_LIBRARY_PATH
+        export PYTHONPATH=$UHD_INSTALL_PATH/lib/$PYTHON_VERSION/site-packages:$PYTHONPATH
+        export PYTHONPATH=$GNURADIO_GRETTUS_INSTALL_PATH/lib/python3/dist-packages:$PYTHONPATH
+      fi
+
+      python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
+        --sdimage_sdmux $(devType),$(devModel),$(uhd_artifact_directory)/$(devType)-images/$(devSDImage),${{ parameters.uhdSrcDir }}/$(devLabgridConfig),$(devHostname) \
+        --fpgas $(devFpga) \
+        --sfp_addrs $(sfpAddrs) \
+        ${{ parameters.redisHost }} $(devName) \
+        "uhd_usrp_probe --args addr=$(devAddr)" \
+        "python3 $(Build.SourcesDirectory)/ettus-rts/config/remote/python_tests/automated_main.py --ats_config $(pytestAtsConfig) --dut $(pytestDUT) \
+          --results_path $(Common.TestResultsDirectory)/pytest/{fpga}"
+    continueOnError: true
+    condition: and(succeeded(), eq(variables.devType, 'n3xx'), eq(variables.devBus, 'ip'))
+    displayName: Run pytests on $(devName)
+
+  - script: |
+      cd $(Common.TestResultsDirectory)/devtest
+      python3 ${{ parameters.uhdSrcDir }}/.ci/utils/format_devtest_junitxml.py \
+        --fpgas $(devFpga) \
+        $(Common.TestResultsDirectory)/pytest \
+        $(Common.TestResultsDirectory)/pytest/pytestresults.xml
+    continueOnError: true
+    displayName: Format devtest xml
+    condition: and(always(), eq(variables.devType, 'n3xx'))
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: '$(Common.TestResultsDirectory)/pytest/pytestresults.xml'
+      testRunTitle: $(devName) pytest
+      buildConfiguration: 'Release'
+      mergeTestResults: true
+    condition: always()
+    displayName: Upload pytest results
+
+  - script: |
+      echo "A previous step failed. See steps with warnings."
+      echo "Marking the overall build status as a failure."
+      exit 1
+    condition: in(variables['Agent.JobStatus'],  'SucceededWithIssues')
+    displayName: Mark build as failure

.ci/templates/stages-uhd-pipeline.yml

@@ -272,6 +272,19 @@ stages:
     parameters:
       testOS: ubuntu1804
 
+- stage: test_uhd_rf_test_stage
+  displayName: Run rf tests n3xx
+  dependsOn:
+  - build_uhd_stage_linux
+  - build_uhd_embedded_system_images
+  - build_gnuradio_stage_linux
+  jobs:
+  - template: job-uhd-rf-tests-pebbles.yml
+    parameters:
+      uhdSrcDir: $(Build.SourcesDirectory)/uhddev
+      testOS: ubuntu1804
+      testDevices: 'n3xx'
+
 - stage: test_streaming_stage
   displayName: Test UHD Streaming
   dependsOn:

.ci/templates/tests/pebbles-labgrid/crossbar/places.yaml

@@ -0,0 +1,17 @@
+pebbles-n310-0:
+  acquired: null
+  acquired_resources: []
+  aliases: []
+  allowed: []
+  changed: 1654034475.1935894
+  comment: ''
+  created: 1654034136.0077882
+  matches:
+  - cls: '*'
+    exporter: '*'
+    group: pebbles-n310-0-group
+    name: null
+    rename: null
+  reservation: null
+  tags: {}
+

.ci/templates/tests/pebbles-labgrid/device-configs/pebbles-n310-0.yml

@@ -0,0 +1,16 @@
+targets:
+  main:
+    resources:
+      RemotePlace:
+        name: 'pebbles-n310-0'
+    drivers:
+    - SerialDriver:
+        name: 'linux_serial_driver'
+        bindings:
+          port: 'console-linux'
+    - SerialDriver:
+        name: 'scu_serial_driver'
+        bindings:
+          port: 'console-scu'
+    - USBSDMuxDriver: {}
+    - USBStorageDriver: {}

.ci/templates/tests/pebbles-labgrid/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3.3'
+services:
+  coordinator:
+    image: "labgrid-coordinator"
+    volumes:
+      - "./crossbar:/home/root/crossbar:Z"
+    tty: true
+    network_mode: "host"
+    restart: unless-stopped
+    command: bash -c "cp /home/root/crossbar/places.yaml /opt/crossbar/places.yaml &&
+      crossbar start --config /opt/labgrid/.crossbar/config.yaml"
+  exporter:
+    image: "labgrid-exporter"
+    volumes:
+      - "./exporter-conf:/opt/conf:Z"
+      - "/run/udev:/run/udev:ro"
+      - "/dev:/dev"
+    depends_on:
+      - coordinator
+    tty: true
+    network_mode: "host"
+    restart: unless-stopped
+    stdin_open: true
+    privileged: true
+    command: bash -c "set -e &&
+      /opt/wait-for-it/wait-for-it.sh 127.0.0.1:20408 -- labgrid-exporter /opt/conf/exporter.yaml"

.ci/templates/tests/pebbles-labgrid/exporter-conf/exporter.yaml

@@ -0,0 +1,16 @@
+pebbles-n310-0-group:
+  console-scu:
+    cls: USBSerialPort
+    match:
+      ID_SERIAL: 'Silicon_Labs_CP2105_Dual_USB_to_UART_Bridge_Controller_0097D46F'
+      ID_USB_INTERFACE_NUM: '01'
+    speed: 115200
+  console-linux:
+    cls: USBSerialPort
+    match:
+      ID_SERIAL: 'Silicon_Labs_CP2105_Dual_USB_to_UART_Bridge_Controller_0097D46F'
+      ID_USB_INTERFACE_NUM: '00'
+    speed: 115200
+  USBSDMuxDevice:
+    match:
+      ID_SERIAL_SHORT: '000000001312'

.ci/uhd-hardware-test-dev.yml

@@ -15,6 +15,10 @@ parameters:
   type: boolean
   displayName: Run devtests
   default: false
+- name: run_rf_tests
+  type: boolean
+  displayName: Run rf tests
+  default: false
 - name: testLength
   type: string
   values:
@@ -84,3 +88,14 @@ stages:
       testOS: ubuntu2004
       uhdSrcDir: $(Build.SourcesDirectory)
       uhdArtifactSource: uhd_mono_pipeline
+
+- stage: test_rf_tests
+  displayName: Run rf tests
+  dependsOn: []
+  condition: ${{ parameters.run_rf_tests }}
+  jobs:
+  - template: templates/job-uhd-rf-tests-pebbles.yml
+    parameters:
+      testOS: ubuntu1804
+      uhdSrcDir: $(Build.SourcesDirectory)/uhddev
+      uhdArtifactSource: uhd_mono_pipeline

.ci/uhd-python-hardware-tests.yml

@@ -1,43 +0,0 @@
-# no CI triggers
-trigger: none
-
-# no PR triggers
-pr: none
-
-parameters:
-- name: run_pebbles_n310_0
-  type: boolean
-  default: true
-- name: runDevTest
-  type: boolean
-  default: false
-- name: runPyTest
-  type: boolean
-  default: true
-
-resources:
-  repositories:
-  - repository: ettus-rts
-    type: github
-    endpoint: EttusResearch
-    name: EttusResearch/ettus-rts
-
-jobs:
-- template: templates/job-uhd-hardware-tests.yml
-  parameters:
-    runDevTest: ${{ parameters.runDevTest }}
-    runPyTest: ${{ parameters.runPyTest }}
-    dutMatrix:
-      pebbles-n310-0:
-        devType: 'n3xx'
-        devModel: 'n310'
-        devName: 'pebbles-n310-0'
-        devSerial: '3176DDF'
-        devBus: ip
-        devAddr: '192.168.40.17'
-        devMgmtAddr: 'ni-n3xx-3176DDF'
-        devtestPattern: 'n3x0'
-        pytestDUT: 'n310-0'
-        pytestAtsConfig: 'uhd_oss_ats'
-        pipelineAgent: pebbles-agent-1
-        deviceEnabled: ${{ parameters.run_pebbles_n310_0 }}

.ci/utils/format_devtest_junitxml.py

@@ -21,7 +21,8 @@ if args.fpgas:
             for case in suite:
                 classname_case = ClassNameTestCase.fromelem(case)
                 if classname_case.name == 'test_all':
-                        classname_case.name = fpga + " " + classname_case.classname
+                    classname_case.name = classname_case.classname
+                classname_case.name = fpga + " " + classname_case.name
         return_xml += xml
     return_xml.write(args.output_name)
 
@@ -34,6 +35,6 @@ else:
         for case in suite:
             classname_case = ClassNameTestCase.fromelem(case)
             if classname_case.name == 'test_all':
-                    classname_case.name = classname_case.classname
+                classname_case.name = classname_case.classname
 
     return_xml.write(args.output_name)

.ci/utils/mutex_hardware.py

@@ -242,7 +242,7 @@ def main(args):
                         set_sfp_addrs(mgmt_addr, sfp_addrs)
 
                 for command in args.test_commands:
-                    result = subprocess.run(shlex.split(command))
+                    result = subprocess.run(shlex.split(command.format(fpga=fpga)))
                     if(return_code == 0):
                         return_code = result.returncode
             sys.exit(return_code)