Skip to content

Commit e055427

Browse files
author
rpennati
committed
[Intel N3000] Refactor and fixes for network preconfig
Fixes and improvements for n3000 scripts: Fecmode without rsu Remove rsu from plugin-init Refactor n3k-mode-manager Invoke n3000-env instead of sourcing it Add setting of routing for tripleo Switch to factory mode in agent-dpdk if user mode is not configured Read fpga image status via sysfs Explicitly verify factory mode in pre-dev-prepare Partial-jira-bug: TFF-24 Signed-off-by: Roberto Pennati <roberto.pennati@hcl.com> Depends-on: Ie49cdd669f9a61e2b702308603886605864f0c70 Change-Id: I8b422d4d9e749eee25e6e5d67e7451f203586939
1 parent fe4f3f2 commit e055427

11 files changed

+198
-121
lines changed

‎containers/vrouter/agent-dpdk/entrypoint.sh

+4-1
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,10 @@ set_ctl net.core.wmem_max 9160000
7373
if [[ -n "${DPDK_UIO_DRIVER}" ]]; then
7474
for i in {0..5} ; do
7575
echo "INFO: Waiting for plugin-init container to place scripts ...($i/5)"
76-
[[ -f "/etc/sysconfig/network-scripts/${DPDK_UIO_DRIVER}/${DPDK_UIO_DRIVER}-pre-dev-prepare-init.sh" ]] && source "/etc/sysconfig/network-scripts/${DPDK_UIO_DRIVER}/${DPDK_UIO_DRIVER}-pre-dev-prepare-init.sh" && break;
76+
if [[ -f "/etc/sysconfig/network-scripts/${DPDK_UIO_DRIVER}/${DPDK_UIO_DRIVER}-pre-dev-prepare-init.sh" ]]; then
77+
source "/etc/sysconfig/network-scripts/${DPDK_UIO_DRIVER}/${DPDK_UIO_DRIVER}-pre-dev-prepare-init.sh"
78+
break
79+
fi
7780
if (( i == 5 )) ; then
7881
echo "WARNING: Time for plugin-init container script placement exceeded."
7982
fi

‎containers/vrouter/plugin/n3000/init/redhat/Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ RUN yes | python3 -m pip install intelhex || true && \
2525
yumdownloader --destdir=/opt/opae/ opae-intel-fpga-driver && \
2626
mkdir -p /opt/n3000/site_packages && \
2727
cp -r /usr/lib/python2.7/site-packages/opae* /opt/n3000/site_packages/ && \
28-
cp /usr/bin/fecmode /opt/n3000/fecmode && \
29-
cp /usr/bin/rsu /opt/n3000/rsu && \
28+
mv /usr/bin/fecmode /opt/n3000/fecmode && \
29+
mv /usr/bin/rsu /opt/n3000/rsu && \
3030
yum clean all -y && \
3131
rm -rf /var/cache/yum
3232

‎containers/vrouter/plugin/n3000/init/redhat/entrypoint.sh

+25-3
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,39 @@ script_dir="/etc/sysconfig/network-scripts/n3000"
44
work_dir="/var/lib/contrail/n3000"
55
lock_file="/var/lib/contrail/n3000/n3000-plugin-init-done"
66

7+
[ -f "${lock_file}" ] && rm "${lock_file}"
8+
79
mkdir -p "${work_dir}/site_packages"
810
mkdir -p "${script_dir}"
911

10-
rm -f /etc/sysconfig/network-scripts/ifcfg-npac*
11-
1212
cp /n3000-* "${script_dir}/"
1313

1414
cp -r /opt/n3000/* "${work_dir}/"
1515

16-
. /n3000-env.sh "${lock_file}" "${work_dir}"
16+
chmod +x /n3000-env.sh
17+
18+
/n3000-env.sh "${lock_file}" "${work_dir}"
19+
20+
chmod +x "${script_dir}/n3000-*"
21+
22+
for npac in $(ip l | awk '/ npac/ { print substr($2,0,length($2)-1); }'); do
23+
cat << EOF > /etc/sysconfig/network-scripts/ifcfg-${npac}
24+
DEVICE=${npac}
25+
ONBOOT=no
26+
HOTPLUG=no
27+
NM_CONTROLLED=no
28+
BOOTPROTO=none
29+
EOF
30+
31+
done
1732

1833
touch "${lock_file}"
1934

2035
echo "INFO: N3000 plugin init container setup done."
36+
37+
echo "DEBUG: /etc/sysconfig/network-scripts directory content:"
38+
echo "-----------------------------------------------------"
39+
40+
find /etc/sysconfig/network-scripts/ -type f -exec ls -al {} +
41+
42+
echo "-----------------------------------------------------"

‎containers/vrouter/plugin/n3000/init/redhat/n3000-common.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
temp_work_dir="/var/run/n3000"
2+
perma_work_dir="/var/lib/contrail/n3000/"
3+
env_file="${perma_work_dir}/n3000-env"
4+
init_lock_file="${perma_work_dir}/n3000-plugin-init-done"
5+
ifcfg_dir="${perma_work_dir}/ifcfgs"
6+
dev_state_restored_lock_file="${perma_work_dir}/n3000-dev-state-restored"

‎containers/vrouter/plugin/n3000/init/redhat/n3000-dev-state-restore.sh

100644100755
+11-6
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
#!/usr/bin/env bash
22

33
source /etc/sysconfig/network-scripts/n3000/n3000-mgmt.sh
4+
source /etc/sysconfig/network-scripts/n3000/n3000-common.sh
45

5-
temp_work_dir="/var/run/n3000"
6-
perma_work_dir="/var/lib/contrail/n3000/"
7-
env_file="${perma_work_dir}/n3000-env"
8-
ifcfg_dir="${perma_data_dir}/ifcfgs"
6+
echo "INFO: n3000-dev-state-restore.sh called"
97

10-
. ${env_file}
8+
[[ -f "${dev_state_restored_lock_file}" ]] && \
9+
echo "INFO: Dev state restore already called for this deinitialization procedure. Skipping next invocation." && exit
10+
11+
touch "${dev_state_restored_lock_file}"
12+
13+
source_env
1114

1215
/etc/sysconfig/network-scripts/n3000/n3000-offload-config.sh "disable" "${temp_work_dir}" "${ifcfg_dir}" "${env_file}"
1316

14-
preconfig_dataplane "noop" "${N3000_CONF}" "${env_file}" "${ifcfg_dir}"
17+
preconfig_dataplane "restore" "${N3000_CONF}" "${env_file}" "${ifcfg_dir}"
18+
19+
rebind_factory "vfio-pci"

‎containers/vrouter/plugin/n3000/init/redhat/n3000-env.sh

100644100755
+18-10
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
11
#/usr/bin/env bash
22

33
source /etc/sysconfig/network-scripts/n3000/n3000-mgmt.sh
4-
5-
lock_file=$1
6-
work_dir=$2
7-
env_file="${work_dir}/n3000-env"
8-
ifcfg_dir="${work_dir}/ifcfgs"
4+
source /etc/sysconfig/network-scripts/n3000/n3000-common.sh
95

106
function propagate_args() {
11-
declare -a env_vars=( N3000_VFs_NUM N3000_VFs_QUEUE_NUM N3000_VDPA_ENABLED N3000_VDPA_MAPPING_ENABLED N3000_INSERT_MODE N3000_PF0_DRIVER N3000_DROP_OFFLOAD_ENABLED N3000_AGING_LCORE_ENABLED N3000_CONF N3000_MODE )
7+
declare -a env_vars=(
8+
N3000_VFs_NUM
9+
N3000_VFs_QUEUE_NUM
10+
N3000_VDPA_ENABLED
11+
N3000_VDPA_MAPPING_ENABLED
12+
N3000_INSERT_MODE
13+
N3000_PF0_DRIVER
14+
N3000_DROP_OFFLOAD_ENABLED
15+
N3000_AGING_LCORE_ENABLED
16+
N3000_CONF
17+
N3000_MODE
18+
N3000_TRIPLEO_L3MH_ROUTE
19+
)
1220

1321
if [[ -f $env_file ]]; then
1422
rm $env_file
@@ -21,13 +29,13 @@ function propagate_args() {
2129
done
2230
}
2331

32+
[[ ! -d "${ifcfg_dir}" ]] && mkdir -p "${ifcfg_dir}"
33+
2434
check_module 'intel-fpga-pci'
2535
check_module 'i40e'
2636

2737
propagate_args
2838

29-
if [[ ! -f "${lock_file}" ]]; then
30-
setup_factory_mode "${env_file}"
39+
setup_factory_mode "${env_file}"
3140

32-
preconfig_dataplane "store" "${N3000_CONF}" "${env_file}" "${ifcfg_dir}"
33-
fi
41+
preconfig_dataplane "store" "${N3000_CONF}" "${env_file}" "${ifcfg_dir}"

‎containers/vrouter/plugin/n3000/init/redhat/n3000-mgmt.sh

+63-49
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
#!/usr/bin/env bash
22

3+
source /etc/sysconfig/network-scripts/n3000/n3000-common.sh
4+
5+
function source_env() {
6+
if [[ -f "${env_file}" ]]; then
7+
echo "INFO: Sourcing env"
8+
echo "INFO: Source env file content:"
9+
echo "$(cat ${env_file})"
10+
. "${env_file}"
11+
fi
12+
}
13+
314
function unbind_driver()
415
{
516
# Expected argument:
@@ -27,15 +38,20 @@ function get_pf0_address() {
2738
}
2839

2940
function get_image_mode {
30-
local env_file=$1
31-
local fpgainfo_file="/usr/bin/fpgainfo"
41+
local fpga_loaded_image_file="/sys/class/fpga/intel-fpga-dev.0/intel-fpga-fme.0/spi-altera.0.auto/spi_master/spi0/spi0.0/fpga_flash_ctrl/fpga_image_load"
42+
43+
if [[ ! -f "${fpga_loaded_image_file}" ]]; then
44+
#TODO: If this happens, device is in bad state
45+
echo -n "ERROR: FPGA bad state"
46+
exit -1
47+
fi
48+
49+
local mode_id="$(cat ${fpga_loaded_image_file})"
3250

33-
if [[ -f "${fpgainfo_file}" ]]; then
34-
# plugin container
35-
echo -n "$(/usr/bin/fpgainfo fme | awk '/Boot Page/ { print $4; }')"
51+
if [[ "${mode_id}" == "0" ]]; then
52+
echo -n "factory"
3653
else
37-
# agent-dpdk container
38-
echo -n "$(cat ${env_file} | awk -F= '/N3000_FPGA_MODE_CONFIGURED/ { print $2; }')"
54+
echo -n "user"
3955
fi
4056
}
4157

@@ -119,7 +135,10 @@ function unbind_n3000_xxv710()
119135
if [[ -d "/sys/bus/pci/devices/${pci_addr}/net" ]]; then
120136
current_device_ifname=$(ls /sys/bus/pci/devices/${pci_addr}/net 2>/dev/null)
121137
if [[ -n "${current_device_ifname}" && -f "/etc/sysconfig/network-scripts/ifcfg-${current_device_ifname}" ]]; then
122-
rm -f /etc/sysconfig/network-scripts/ifcfg-"${current_device_ifname}"
138+
echo "INFO: Moving ifcfg for ${current_device_ifname} to ${ifcfg_dir}"
139+
140+
[[ ! -f "${ifcfg_dir}/ifcfg-${current_device_ifname}" ]] && \
141+
mv -f "/etc/sysconfig/network-scripts/ifcfg-${current_device_ifname}" "${ifcfg_dir}/ifcfg-${current_device_ifname}"
123142
fi
124143
fi
125144
unbind_driver "${pci_addr}"
@@ -173,19 +192,6 @@ function get_config() {
173192
declare -p config
174193
}
175194

176-
function store_ifcfg() {
177-
local ifcfg_dir=$1
178-
local ifname=$2
179-
180-
if [[ ! -f "/etc/sysconfig/network-scripts/ifcfg-$ifname" ]]; then
181-
echo "WARNING: /etc/sysconfig/network-scripts/ifcfg-$ifname not found"
182-
fi
183-
184-
[[ ! -d "$ifcfg_dir" ]] && mkdir -p "$ifcfg_dir"
185-
186-
cp "/etc/sysconfig/network-scripts/ifcfg-$ifname" "$ifcfg_dir/ifcfg-$ifname"
187-
}
188-
189195
function restore_ifcfg() {
190196
local ifcfg_dir=$1
191197

@@ -200,11 +206,13 @@ function restore_ifcfg() {
200206
local ifile="$(echo $ifcfg_file | awk -F/ '{ print $NF; }')"
201207

202208
if [[ -f "/etc/sysconfig/network-scripts/$ifile" ]]; then
203-
echo "WARNING: /etc/sysconfig/network-scripts/$ifile exists - should be absent at this point"
209+
echo "INFO: /etc/sysconfig/network-scripts/$ifile exists - skipping"
204210
continue
205211
fi
206212

207213
cp "$ifcfg_file" "/etc/sysconfig/network-scripts/$ifile"
214+
215+
echo "INFO: /etc/sysconfig/network-scripts/$ifile restored"
208216
done
209217

210218
shopt -u nullglob
@@ -232,16 +240,14 @@ function preconfig_dataplane() {
232240
local ifname_key="phy${i}_ifname"
233241
local requested_ifname="${config[${ifname_key}]}"
234242
echo "INFO: phy_addr=$phy_addr, ifname found: $phy_ifname, requested ifname: $requested_ifname"
235-
236-
if [[ "$phy_ifname" != "$requested_ifname" ]]; then
237-
ip link set dev $phy_ifname name $requested_ifname
238-
fi
243+
ip link set "${phy_ifname}" down
244+
ip link set "${phy_ifname}" name "${requested_ifname}"
239245

240246
set_network_info_for_device "${requested_ifname}" "${phy_addr}"
241247

242-
/usr/sbin/ifup $requested_ifname
248+
[ -n "${N3000_TRIPLEO_L3MH_ROUTE}" ] && echo "${N3000_TRIPLEO_L3MH_ROUTE}" > "/etc/sysconfig/network-scripts/route-${requested_ifname}"
243249

244-
[[ "${ifcfg_op}" != "noop" ]] && store_ifcfg "$ifcfg_dir" "$requested_ifname"
250+
/usr/sbin/ifup $requested_ifname
245251
done
246252
}
247253

@@ -253,15 +259,18 @@ function preconfig_dataplane() {
253259
if [[ $phy_id == "0" || $phy_id == "1" ]]; then
254260
local phy_addr="$(lspci -d :0d58 -D | awk '{print $1}' | awk "NR == $(( phy_id + 1 ))")"
255261
local phy_ifname=$(realpath -e /sys/bus/pci/devices/${phy_addr}/net/* 2>/dev/null | awk -F/ '{print $NF}')
262+
local requested_ifname="${PHYSICAL_INTERFACE}"
263+
if [[ -z "${PHYSICAL_INTERFACE}" ]]; then
264+
requested_ifname="$(cat /var/run/vrouter/nic)"
265+
fi
256266

267+
echo "INFO: phy_addr=$phy_addr, ifname found: $phy_ifname, requested ifname: $requested_ifname"
257268
ip link set "${phy_ifname}" down
258-
ip link set "${phy_ifname}" name "${PHYSICAL_INTERFACE}"
269+
ip link set "${phy_ifname}" name "${requested_ifname}"
259270

260-
set_network_info_for_device "${PHYSICAL_INTERFACE}" "${phy_addr}"
261-
262-
ip link set "${PHYSICAL_INTERFACE}" up
271+
set_network_info_for_device "${requested_ifname}" "${phy_addr}"
263272

264-
[[ "${ifcfg_op}" != "noop" ]] && store_ifcfg "$ifcfg_dir" "${PHYSICAL_INTERFACE}"
273+
ip link set "${requested_ifname}" up
265274
fi
266275
}
267276

@@ -270,15 +279,18 @@ function preconfig_dataplane() {
270279

271280
local phy_addr="$(lspci -d :0d58 -D | awk '{print $1}' | awk "NR == 1")"
272281
local phy_ifname=$(realpath -e /sys/bus/pci/devices/${phy_addr}/net/* 2>/dev/null | awk -F/ '{print $NF}')
282+
local requested_ifname="${PHYSICAL_INTERFACE}"
283+
if [[ -z "${PHYSICAL_INTERFACE}" ]]; then
284+
requested_ifname="$(cat /var/run/vrouter/nic)"
285+
fi
273286

287+
echo "INFO: phy_addr=$phy_addr, ifname found: $phy_ifname, requested ifname: $requested_ifname"
274288
ip link set "${phy_ifname}" down
275-
ip link set "${phy_ifname}" name "${PHYSICAL_INTERFACE}"
276-
277-
set_network_info_for_device "${PHYSICAL_INTERFACE}" "${phy_addr}"
289+
ip link set "${phy_ifname}" name "${requested_ifname}"
278290

279-
ip link set "${PHYSICAL_INTERFACE}" up
291+
set_network_info_for_device "${requested_ifname}" "${phy_addr}"
280292

281-
[[ "${ifcfg_op}" != "noop" ]] && store_ifcfg "$ifcfg_dir" "${PHYSICAL_INTERFACE}"
293+
ip link set "${requested_ifname}" up
282294
}
283295

284296
local ifcfg_op=$1
@@ -296,6 +308,8 @@ function preconfig_dataplane() {
296308
echo "INFO: Config:"
297309
declare -p config
298310

311+
[[ "${ifcfg_op}" == "restore" ]] && restore_ifcfg "$ifcfg_dir"
312+
299313
prepare_$config_type
300314
}
301315

@@ -340,7 +354,7 @@ function verify_user_mode_config {
340354
function switch_fpga_mode() {
341355
local requested_mode=$1
342356
local env_file=$2
343-
local found_mode=$(get_image_mode "${env_file}")
357+
local found_mode="$(get_image_mode)"
344358

345359
if [[ "${found_mode}" == "user" ]]; then
346360
local user_mode_config="$(verify_user_mode_config)"
@@ -358,26 +372,26 @@ function switch_fpga_mode() {
358372
# However the deployment logic requires to only switch between modes -
359373
# that is: factory to known user mode state and unknown user mode to factory mode.
360374
# so there is no need for a logic for unknown user mode state to known user mode state
375+
echo "INFO: ${requested_mode} mode already present"
361376
return
362377
fi
363378

364379
N3000_SWITCH_MODE="${requested_mode}" /etc/sysconfig/network-scripts/n3000/n3000-mode-manager.sh
380+
}
365381

366-
if [[ -n "$(grep N3000_FPGA_MODE_CONFIGURED ${env_file})" ]]; then
367-
sed -i "s/N3000_FPGA_MODE_CONFIGURED=*/N3000_FPGA_MODE_CONFIGURED=${requested_mode}/g" ${env_file}
368-
else
369-
echo "N3000_FPGA_MODE_CONFIGURED=${requested_mode}" >> ${env_file}
370-
fi
382+
function rebind_factory {
383+
local driver=$1
384+
for i in $(seq 1 2); do
385+
phy_addr=$( lspci -d :0d58 -D | awk '{print $1}' | awk "NR == $i" )
386+
387+
override_driver "${phy_addr}" "${driver}"
388+
done
371389
}
372390

373391
function setup_factory_mode {
374392
local env_file=$1
375393

376394
switch_fpga_mode "factory" "${env_file}"
377395

378-
for i in $(seq 1 2); do
379-
phy_addr=$( lspci -d :0d58 -D | awk '{print $1}' | awk "NR == $i" )
380-
381-
override_driver "${phy_addr}" "i40e"
382-
done
396+
rebind_factory "i40e"
383397
}

0 commit comments

Comments
 (0)