ci: Add n310 into devtests

This commit adds support for reimaging n3xx devices and running
devtest against a n310. This uses Labgrid to control an sdmux,
which we then reimage the device. In the future, other devices can
use this support to run other tests.

Signed-off-by: Steven Koo <steven.koo@ni.com>
This commit is contained in:
Steven Koo 2022-06-14 09:27:41 -05:00 committed by skooNI
parent ce000955ae
commit 8d05b1cda1
8 changed files with 289 additions and 83 deletions

View File

@ -8,6 +8,9 @@ parameters:
- name: uhdArtifactSource
type: string
default: current
- name: testDevices
type: string
default: 'x3xx,b2xx,n3xx'
jobs:
- template: job-uhd-devtest.yml
@ -20,77 +23,94 @@ jobs:
uhdArtifactSource: '${{ parameters.uhdArtifactSource }}'
redisHost: 'sdr-rhombus'
dutMatrix:
rhombus-x300-UBX-0:
devType: 'x300'
devModel: 'x300'
devName: 'rhombus-x300-UBX-0'
devSerial: '30A6019'
devBus: 'ip'
devAddr: '192.168.40.2'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '2516350A6019'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x310-UBX-0:
devType: 'x300'
devModel: 'x310'
devName: 'rhombus-x310-UBX-0'
devSerial: '3138EF5'
devBus: 'ip'
devAddr: '192.168.40.3'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '251635138E98'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x310-CBX-0:
devType: 'x300'
devModel: 'x310'
devName: 'rhombus-x310-CBX-0'
devSerial: '30796C2'
devBus: 'ip'
devAddr: '192.168.40.4'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '2516350796C2'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x310-WBX-0:
devType: 'x300'
devModel: 'x310'
devName: 'rhombus-x310-WBX-0'
devSerial: '30C5BFF'
devBus: 'ip'
devAddr: '192.168.40.5'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '2516350C5BFF'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x310-TWINRX-0:
devType: 'x300'
devModel: 'x310'
devName: rhombus-x310-TWINRX-0
devSerial: 'F43D13'
devBus: 'ip'
devAddr: '192.168.40.6'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '251635F43D13'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x300-SBX-0:
devType: 'x300'
devModel: 'x300'
devName: rhombus-x300-SBX-0
devSerial: '32244AD'
devBus: 'ip'
devAddr: '192.168.40.7'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '2516352244AD'
jtagServer: 'nitest@sdr-rhombus'
${{ if contains(parameters.testDevices, 'x3xx') }}:
rhombus-x300-UBX-0:
devType: 'x300'
devModel: 'x300'
devName: 'rhombus-x300-UBX-0'
devSerial: '30A6019'
devBus: 'ip'
devAddr: '192.168.40.2'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '2516350A6019'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x310-UBX-0:
devType: 'x300'
devModel: 'x310'
devName: 'rhombus-x310-UBX-0'
devSerial: '3138EF5'
devBus: 'ip'
devAddr: '192.168.40.3'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '251635138E98'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x310-CBX-0:
devType: 'x300'
devModel: 'x310'
devName: 'rhombus-x310-CBX-0'
devSerial: '30796C2'
devBus: 'ip'
devAddr: '192.168.40.4'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '2516350796C2'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x310-WBX-0:
devType: 'x300'
devModel: 'x310'
devName: 'rhombus-x310-WBX-0'
devSerial: '30C5BFF'
devBus: 'ip'
devAddr: '192.168.40.5'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '2516350C5BFF'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x310-TWINRX-0:
devType: 'x300'
devModel: 'x310'
devName: rhombus-x310-TWINRX-0
devSerial: 'F43D13'
devBus: 'ip'
devAddr: '192.168.40.6'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '251635F43D13'
jtagServer: 'nitest@sdr-rhombus'
rhombus-x300-SBX-0:
devType: 'x300'
devModel: 'x300'
devName: rhombus-x300-SBX-0
devSerial: '32244AD'
devBus: 'ip'
devAddr: '192.168.40.7'
devFpga: 'HG,XG'
devtestPattern: 'x3x0'
jtagSerial: '2516352244AD'
jtagServer: 'nitest@sdr-rhombus'
rhombus-b210-0:
devType: 'b200'
devModel: 'b210'
devName: rhombus-b210-0
devSerial: '3218D0E'
devBus: 'usb'
devtestPattern: 'b2xx'
${{ if contains(parameters.testDevices, 'b2xx') }}:
rhombus-b210-0:
devType: 'b200'
devModel: 'b210'
devName: rhombus-b210-0
devSerial: '3218D0E'
devBus: 'usb'
devtestPattern: 'b2xx'
${{ if contains(parameters.testDevices, 'n3xx') }}:
rhombus-n310-0:
devType: 'n3xx'
devModel: 'n310'
devName: rhombus-n310-0
devSerial: '3176DF3'
devHostname: 'ni-n3xx-3176df3'
devBus: 'ip'
devAddr: '192.168.20.5'
sfpAddrs: '192.168.10.5,192.168.20.5'
devFpga: 'HG,XG'
devtestPattern: 'n3x0'
devSDImage: gnuradio-image-ni-sulfur-rev11-mender.sdimg.bz2
devLabgridConfig: .ci/templates/tests/rhombus-labgrid/device-configs/rhombus-n310-0.yml

View File

@ -65,6 +65,14 @@ jobs:
destinationFolder: $(Build.BinariesDirectory)
cleanDestinationFolder: true
- download: ${{ parameters.uhdArtifactSource }}
artifact: $(devType)-images
# Only sync the bz2 sdimg since the bmap
# is incompatible with mender
patterns: '**/*.bz2'
displayName: Download $(devType)-images artifact
condition: and(succeeded(), eq(variables.devType, 'n3xx'))
- script: |
cd $(Build.BinariesDirectory)/uhddev/build
mkdir -p fpga_images
@ -77,6 +85,27 @@ jobs:
fi
displayName: Download FPGA Images
- script: |
mkdir -p $(Common.TestResultsDirectory)/devtest
cd $(Common.TestResultsDirectory)/devtest
export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
export UHD_IMAGES_DIR=$(Build.BinariesDirectory)/uhddev/build/fpga_images
python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
--sdimage $(devType),$(devModel),$(uhd_artifact_directory)/$(devType)-images/$(devSDImage),${{ parameters.uhdSrcDir }}/$(devLabgridConfig),$(devHostname) \
--fpgas $(devFpga) \
--sfp_addrs $(sfpAddrs) \
${{ parameters.redisHost }} $(devName) \
"$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args addr=$(devAddr)" \
"python3 ${{ parameters.uhdSrcDir }}/host/tests/devtest/run_testsuite.py \
--src-dir ${{ parameters.uhdSrcDir }}/host/tests/devtest \
--devtest-pattern $(devtestPattern) --args addr=$(devAddr),type=$(devType) \
--build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \
--python-interp python3 --xml"
continueOnError: true
condition: and(succeeded(), eq(variables.devType, 'n3xx'), eq(variables.devBus, 'ip'))
displayName: Run n3xx devtest on $(devName)
- script: |
mkdir -p $(Common.TestResultsDirectory)/devtest
cd $(Common.TestResultsDirectory)/devtest
@ -102,7 +131,7 @@ jobs:
export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
--jtag_x3xx $(jtagServer),$(jtagSerial),$(devModel),$(Build.BinariesDirectory)/uhddev/build/fpga_images/ \
--jtag_x3xx $(devType),$(devModel),$(jtagServer),$(jtagSerial),$(Build.BinariesDirectory)/uhddev/build/fpga_images/ \
--fpgas $(devFpga) \
${{ parameters.redisHost }} $(devName) \
"$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args addr=$(devAddr)" \

View File

@ -226,17 +226,31 @@ stages:
toolset: msbuild
installer: nsis
- stage: test_uhd_stage
displayName: Test UHD
dependsOn: build_uhd_stage_linux
- stage: devtest_uhd_x3xx_b2xx_stage
displayName: devtest UHD x3xx b2xx
dependsOn:
- build_uhd_stage_linux
jobs:
- template: job-uhd-devtest-rhombus.yml
parameters:
testOS: ubuntu2004
uhdSrcDir: $(Build.SourcesDirectory)
testDevices: 'x3xx,b2xx'
- stage: test_uhd_embedded_devices_stage
displayName: Test UHD Embedded Devices
- stage: devtest_uhd_n3xx_stage
displayName: devtest UHD n3xx
dependsOn:
- build_uhd_stage_linux
- build_uhd_embedded_system_images
jobs:
- template: job-uhd-devtest-rhombus.yml
parameters:
testOS: ubuntu2004
uhdSrcDir: $(Build.SourcesDirectory)
testDevices: 'n3xx'
- stage: test_uhd_x4xx_stage
displayName: Test UHD x4xx
dependsOn:
- build_uhd_stage_linux
- build_uhd_embedded_system_images

View File

@ -0,0 +1,17 @@
rhombus-n310-0:
acquired: null
acquired_resources: []
aliases: []
allowed: []
changed: 1654034475.1935894
comment: ''
created: 1654034136.0077882
matches:
- cls: '*'
exporter: '*'
group: rhombus-n310-0-group
name: null
rename: null
reservation: null
tags: {}

View File

@ -0,0 +1,16 @@
targets:
main:
resources:
RemotePlace:
name: 'rhombus-n310-0'
drivers:
- SerialDriver:
name: 'linux_serial_driver'
bindings:
port: 'console-linux'
- SerialDriver:
name: 'scu_serial_driver'
bindings:
port: 'console-scu'
- USBSDMuxDriver: {}
- USBStorageDriver: {}

View File

@ -0,0 +1,26 @@
version: '3.3'
services:
coordinator:
image: "labgrid-coordinator"
volumes:
- "./crossbar:/home/root/crossbar:Z"
tty: true
network_mode: "host"
restart: unless-stopped
command: bash -c "cp /home/root/crossbar/places.yaml /opt/crossbar/places.yaml &&
crossbar start --config /opt/labgrid/.crossbar/config.yaml"
exporter:
image: "labgrid-exporter"
volumes:
- "./exporter-conf:/opt/conf:Z"
- "/run/udev:/run/udev:ro"
- "/dev:/dev"
depends_on:
- coordinator
tty: true
network_mode: "host"
restart: unless-stopped
stdin_open: true
privileged: true
command: bash -c "set -e &&
/opt/wait-for-it/wait-for-it.sh 127.0.0.1:20408 -- labgrid-exporter /opt/conf/exporter.yaml"

View File

@ -0,0 +1,17 @@
rhombus-n310-0-group:
console-scu:
cls: USBSerialPort
match:
ID_SERIAL: 'Silicon_Labs_CP2105_Dual_USB_to_UART_Bridge_Controller_0097D446'
ID_USB_INTERFACE_NUM: '01'
speed: 115200
console-linux:
cls: USBSerialPort
match:
ID_SERIAL: 'Silicon_Labs_CP2105_Dual_USB_to_UART_Bridge_Controller_0097D446'
ID_USB_INTERFACE_NUM: '00'
speed: 115200
USBSDMuxDevice:
match:
ID_SERIAL_SHORT: '000000001506'

View File

@ -4,6 +4,7 @@
# into a state where it can be used for testing.
import argparse
import labgrid
import os
import pathlib
import shlex
@ -17,7 +18,7 @@ from redis import Redis
bitfile_name = "usrp_{}_fpga_{}.bit"
def jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_server):
def jtag_x3xx(dev_type, dev_model, jtag_server, jtag_serial, fpga_folder, fpga, redis_server):
remote_working_dir = "pipeline_fpga"
vivado_program_jtag = "/opt/Xilinx/Vivado_Lab/2020.1/bin/vivado_lab -mode batch -source {}/viv_hardware_utils.tcl -nolog -nojournal -tclargs program".format(
remote_working_dir)
@ -40,12 +41,69 @@ def jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_serv
print("Waiting 15 seconds for device to come back up and for Vivado to close", flush=True)
time.sleep(15)
def set_sfp_addrs(mgmt_addr, sfp_addrs):
with Connection(host=mgmt_addr,user='root',connect_kwargs={"password":"", "timeout":120, "banner_timeout":120, "auth_timeout":120}) as dut:
for idx, sfp_addr in enumerate(sfp_addrs):
dut.run(f"ip link set sfp{idx} down")
dut.run(f"ip addr add {sfp_addr}/24 dev sfp{idx}")
dut.run(f"ip link set sfp{idx} up")
time.sleep(30)
def flash_sdimage(dev_model, sdimage_path, labgrid_device_yaml, mgmt_addr, sfp_addrs):
subprocess.run(shlex.split(f"labgrid-client -c {labgrid_device_yaml} release --kick"))
subprocess.run(shlex.split(f"labgrid-client -c {labgrid_device_yaml} acquire"))
env = labgrid.Environment(labgrid_device_yaml)
target = env.get_target()
print("Powering down DUT", flush=True)
cp = target.get_driver(labgrid.protocol.ConsoleProtocol, name="scu_serial_driver")
cp.write("\napshutdown\n".encode())
print("Switching SDMux to Host", flush=True)
sdmux = target.get_driver(labgrid.driver.USBSDMuxDriver)
sdmux.set_mode('host')
print(f"Writing SDMux using {sdimage_path}", flush=True)
massstore = target.get_driver(labgrid.driver.USBStorageDriver)
# This uses bmaptool in --nobmap mode to write the sdimg,
# since it automatically decompresses the image. Do not
# include the .bmap file since it can cause corruption with mender.
# See: https://github.com/mendersoftware/meta-mender/pull/1076
massstore.write_image(sdimage_path, mode=labgrid.driver.usbstoragedriver.Mode.BMAPTOOL)
time.sleep(30)
print("Switching SDMux to DUT", flush=True)
sdmux.set_mode('dut')
time.sleep(30)
print("Powering on DUT", flush=True)
cp.write("\npowerbtn\n".encode())
print("Waiting 2 minutes for device to boot", flush=True)
time.sleep(120)
known_hosts_path = os.path.expanduser("~/.ssh/known_hosts")
subprocess.run(shlex.split(f"ssh-keygen -f \"{known_hosts_path}\" -R \"{mgmt_addr}\""))
if sfp_addrs:
set_sfp_addrs(mgmt_addr, sfp_addrs)
subprocess.run(shlex.split(f"labgrid-client -c {labgrid_device_yaml} release"))
def main(args):
redis_server = {Redis.from_url(
"redis://{}:6379/0".format(args.redis_server))}
print("Waiting to acquire mutex for {}".format(args.dut_name), flush=True)
with Redlock(key=args.dut_name, masters=redis_server, auto_release_time=1000 * 60 * args.dut_timeout):
print("Got mutex for {}".format(args.dut_name), flush=True)
if args.sdimage:
dev_type, dev_model, sdimage_path, labgrid_device_yaml, mgmt_addr = args.sdimage.split(',')
if args.sfp_addrs:
sfp_addrs = args.sfp_addrs.split(',')
else:
sfp_addrs = None
flash_sdimage(dev_model, sdimage_path, labgrid_device_yaml, mgmt_addr, sfp_addrs)
if args.fpgas:
working_dir = os.getcwd()
return_code = 0
@ -54,8 +112,13 @@ def main(args):
os.chdir(os.path.join(working_dir, fpga))
if args.jtag_x3xx:
jtag_server, jtag_serial, dev_model, fpga_folder = args.jtag_x3xx.split(',')
jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_server)
dev_type, dev_model, jtag_server, jtag_serial, fpga_folder = args.jtag_x3xx.split(',')
jtag_x3xx(dev_type, dev_model, jtag_server, jtag_serial, fpga_folder, fpga, redis_server)
if dev_type and dev_type in ["n3xx", "e3xx"]:
subprocess.run(shlex.split(f"uhd_image_loader --args=mgmt_addr={mgmt_addr},type={dev_type},fpga={fpga}"))
if sfp_addrs:
set_sfp_addrs(mgmt_addr, sfp_addrs)
for command in args.test_commands:
result = subprocess.run(shlex.split(command))
@ -76,7 +139,11 @@ if __name__ == "__main__":
# Provide fpga_path as a local path and it will be copied
# to jtag_server.
parser.add_argument("--jtag_x3xx", type=str,
help="user@jtag_server,jtag_serial,dev_model,fpga_folder")
help="dev_type,dev_model,user@jtag_server,jtag_serial,fpga_folder")
parser.add_argument("--sdimage", type=str,
help="dev_type,dev_model,sdimg_path,labgrid_device_yaml,mgmt_addr")
parser.add_argument("--sfp_addrs", type=str,
help="sfp0ip,sfp1ip,...")
parser.add_argument("--fpgas", type=str,
help="Comma delimited list of FPGAs to test")
parser.add_argument("--dut_timeout", type=int, default=60,