Switch to use upstream repos from Juniper

- move cloned repositories do vars dir in checkout_repos role
- clone all repositories contrail-dev-env depends on in checkout_repos target
- use Zuulv3 like directory layout for source code
- change Vagrantfile to mirror contrail-dev-env playbooks changes

Author: pjrusak

Makefile

@@ -1,39 +1,41 @@
 sandbox_path=$(HOME)/contrail-5.0
+repos_dir=$(HOME)/src/review.opencontrail.org/Juniper/
 ansible_playbook=ansible-playbook -i inventory --extra-vars @vars.yaml --extra-vars @dev_config.yaml
 
 .PHONY: presetup checkout_vnc setup build rpm containers deploy unittest sanity all
 
 # this is the first bootstrap of the packages for the tool itself
 # not a part of the "all" target, should be invoked manually
 presetup:
-	yum install -y epel-release ansible vim
+	yum install -y epel-release ansible git vim
 
 # optional step, used when the sandbox is not mounted from host system
-checkout_vnc: setup
+checkout_repos:
+	$(ansible_playbook) provisioning/checkout_repos.yaml
 	scripts/checkout_vnc.sh $(sandbox_path)
 
 # install all the primary build deps, docker engine etc.
-setup:
-	$(ansible_playbook) provisioning/setup1.yaml
-	sudo ansible-playbook -e '{"CREATE_CONTAINERS":false, "CONTAINER_VM_CONFIG": {"network": {"ntpserver":"127.0.0.1"}}, "CONTAINER_REGISTRY": "172.17.0.1:6666", "REGISTRY_PRIVATE_INSECURE": true, "CONFIGURE_VMS":true, "roles": {"localhost":[]}}' -i inventory -c local  code/contrail-ansible-deployer/playbooks/deploy.yml
-	$(ansible_playbook) provisioning/setup2.yaml
+setup: checkout_repos
+	$(ansible_playbook) provisioning/setup_vm.yaml
+	sudo ansible-playbook -e '{"CREATE_CONTAINERS":false, "CONTAINER_VM_CONFIG": {"network": {"ntpserver":"127.0.0.1"}}, "contrail_configuration": {"CONTAINER_REGISTRY": "172.17.0.1:6666"}, "REGISTRY_PRIVATE_INSECURE": true, "CONFIGURE_VMS":true, "roles": {"localhost":[]}}' -i inventory -c local  $(repos_dir)/contrail-ansible-deployer/playbooks/deploy.yml
+	$(ansible_playbook) provisioning/complete_vm_config.yaml
 
 build:
 	echo "Not implemented yet"
 
 rpm: setup
-	scripts/setup_build_logging.sh
-	$(ansible_playbook) code/contrail-project-config/playbooks/packaging/contrail-vnc-el.yaml
+	scripts/setup_build_logging.sh $(repos_dir)
+	$(ansible_playbook) $(repos_dir)/contrail-project-config/playbooks/packaging/contrail-vnc-el.yaml
 	createrepo $(HOME)/rpmbuild/RPMS/
 
 containers: rpm
-	$(ansible_playbook) code/contrail-project-config/playbooks/docker/centos74.yaml
+	$(ansible_playbook) $(repos_dir)/contrail-project-config/playbooks/docker/centos74.yaml
 
 deploy_contrail_kolla: containers
-	$(ansible_playbook) code/contrail-project-config/playbooks/kolla/centos74-provision-kolla.yaml
+	$(ansible_playbook) $(repos_dir)/contrail-project-config/playbooks/kolla/centos74-provision-kolla.yaml
 
 deploy_contrail_k8s: containers
-	$(ansible_playbook) code/contrail-project-config/playbooks/docker/centos74-systest-kubernetes.yaml
+	$(ansible_playbook) $(repos_dir)/contrail-project-config/playbooks/docker/centos74-systest-kubernetes.yaml
 
 unittests: build
 	echo "Not implemented yet"

Vagrantfile

@@ -2,18 +2,24 @@
 # vi: set ft=ruby :
 
 VAGRANTFILE_API_VERSION = "2"
-DEV_ENV_DIR="/root/contrail-dev-env"
+USER = "root"
+GROUP = "root"
+HOME_DIR = "/root"
+DEV_ENV_DIR = "#{HOME_DIR}/contrail-dev-env"
+REPOS_DIR = "#{HOME_DIR}/src/review.opencontrail.org/Juniper/"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
    config.vm.define :contrail_sandbox do |contrail_sandbox_config|
       contrail_sandbox_config.vm.box = "geerlingguy/centos7"
       contrail_sandbox_config.vm.hostname = "contrail-sandbox"
       contrail_sandbox_config.vm.network "private_network", ip: "192.168.60.200"
-      contrail_sandbox_config.vm.synced_folder ".", DEV_ENV_DIR
+      contrail_sandbox_config.vm.synced_folder "./code", "#{REPOS_DIR}",
+	      owner: "#{USER}", group: "#{GROUP}"
+      contrail_sandbox_config.vm.synced_folder ".", "#{DEV_ENV_DIR}", owner: "#{USER}", group: "#{GROUP}"
       contrail_sandbox_config.ssh.forward_agent = true
-      contrail_sandbox_config.ssh.insert_key = 'true'
-      contrail_sandbox_config.ssh.username = 'root'
+      contrail_sandbox_config.ssh.insert_key = true
+      contrail_sandbox_config.ssh.username = "#{USER}"
       contrail_sandbox_config.ssh.password = 'vagrant'
 
       contrail_sandbox_config.vm.provider "virtualbox" do |vb|
@@ -22,7 +28,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       end
       contrail_sandbox_config.vm.provision "shell", name: "run presetup" do |presetup|
         presetup.inline = "make -f $1/Makefile presetup"
-        presetup.args = DEV_ENV_DIR
+        presetup.args = "#{DEV_ENV_DIR}"
       end
    end
 end

provisioning/checkout_repos.yaml

@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - role: checkout_repos

provisioning/setup2.yaml renamed to provisioning/complete_vm_config.yaml

@@ -1,12 +1,24 @@
+- name: Download Android repo tool
+  get_url:
+    url: "https://storage.googleapis.com/git-repo-downloads/repo"
+    dest: "/usr/bin/repo"
+    mode: 0655
+  become: yes
+  become_user: root
+
+- name: Prepare Zuulv3 like dir hierarchy for code
+  file:
+    path: "{{ workdir }}/src/review.opencontrail.org/Juniper/"
+    state: directory
+    owner: "{{ sandbox_owner }}"
+    group: "{{ sandbox_group }}"
+  register: repos_dir
+
 - name: Checkout CI jobs repos
   git:
-    repo: "https://github.com/{{ item.user }}/{{ item.reponame }}"
-    dest: "{{ workdir }}/contrail-dev-env/code/{{ item.reponame }}"
+    repo: "https://github.com/{{ item.repouser }}/{{ item.reponame }}"
+    dest: "{{ repos_dir.path }}//{{ item.reponame }}"
     version: "{{ branch | default('master') }}"
     update: "{{ update_repos | default('no') }}"
     force: "{{ override_repos | default('no') }}"
-  with_items:
-    - { user: 'codilime', reponame: 'contrail-project-config' }
-    - { user: 'codilime', reponame: 'zuul-jobs' }
-    - { user: 'Juniper', reponame: 'contrail-ansible-deployer' }
-
+  with_items: "{{ cloned_repos }}"

provisioning/roles/checkout_repos/tasks/main.yaml

@@ -0,0 +1,9 @@
+cloned_repos:
+  - reponame: 'contrail-project-config'
+    repouser: 'Juniper'
+  - reponame: 'zuul-jobs'
+    repouser: 'Juniper'
+  - reponame: 'contrail-ansible-deployer'
+    repouser: 'Juniper'
+  - reponame: 'contrail-container-builder'
+    repouser: 'Juniper'

provisioning/roles/checkout_repos/vars/main.yaml

@@ -3,21 +3,10 @@
     name: "{{ item }}"
     state: present
   with_items:
-    - epel-release
-    - ansible
-    - git
     - createrepo
     - tree
     - lynx
     - docker-python
   become: yes
   become_user: root
 
-- name: Download Android repo tool
-  get_url:
-    url: "https://storage.googleapis.com/git-repo-downloads/repo"
-    dest: "/usr/bin/repo"
-    mode: 0655
-  become: yes
-  become_user: root
-

provisioning/roles/generic_dependencies/tasks/main.yaml

@@ -1,5 +1,5 @@
 [defaults]
-{% set repodir = workdir + '/contrail-dev-env/code' %}
+{% set repodir = workdir + '/src/review.opencontrail.org/Juniper' %}
 roles_path = {{ repodir }}/contrail-project-config/roles/:{{ repodir }}/zuul-jobs/roles/
 nocows = 1
 retry_files_enabled = False

provisioning/roles/setup_sandbox/templates/ansible.cfg.j2

@@ -3,5 +3,3 @@
     - role: generic_dependencies
     - role: build_dependencies
     - role: setup_sandbox
-    - role: checkout_repos
-

provisioning/setup1.yaml renamed to provisioning/setup_vm.yaml

@@ -0,0 +1,11 @@
+--- build_package.yaml.orig	2018-02-12 13:42:54.150293579 +0000
++++ build_package.yaml	2018-02-12 13:48:42.860086032 +0000
+@@ -6,7 +6,8 @@
+       --define "_srcVer {{ packaging.version.upstream }}" \
+       --define "_skuTag mitaka" \
+       {{ item.extra|default('') }} \
+-      -bb {{ ansible_env.HOME }}/{{ packaging.target_dir }}/tools/packaging/common/rpm/{{ item.name }}.spec
++      -bb {{ ansible_env.HOME }}/{{ packaging.target_dir }}/tools/packaging/common/rpm/{{ item.name }}.spec \
++      |& tee {{ ansible_env.HOME }}/contrail-build-logs/contrail-logs/{{ item.name }}_$(date "+%Y%m%d_%H%M%S").log
+   args:
+     chdir: "{{ ansible_env.HOME }}/{{ packaging.target_dir }}/{{ item.workdir|default('') }}"

scripts/build_package.patch

@@ -1,8 +1,8 @@
---- code/contrail-project-config/roles/packaging-build-el/tasks/main.yaml	2018-02-01 08:28:29.997194231 +0000
-+++ main.yaml.new	2018-02-01 08:28:16.613354996 +0000
+--- main.yaml.orig	2018-02-12 13:42:46.830173553 +0000
++++ main.yaml	2018-02-12 13:47:38.719011505 +0000
 @@ -22,7 +22,8 @@
        --define "_kVers {{ kver.stdout }}" \
-       --define "_buildTag {{ zuul.change }}.{{ zuul.patchset }}" \
+       --define "_buildTag {{ packaging.version.distrib }}" \
        --define "_srcVer {{ packaging.version.upstream }}" \
 -      -ba tools/packages/rpm/contrail/contrail.spec
 +      -ba tools/packages/rpm/contrail/contrail.spec \
@@ -12,7 +12,7 @@
    environment:
 @@ -35,7 +36,8 @@
        --define '_builddir {{ ansible_env.HOME }}/{{ packaging.target_dir }}/controller' \
-       --define "_buildTag {{ zuul.change }}.{{ zuul.patchset }}" \
+       --define "_buildTag {{ packaging.version.distrib }}" \
        --define "_srcVer {{ packaging.version.upstream }}" \
 -      -ba {{ ansible_env.HOME }}/{{ packaging.target_dir }}/tools/packaging/common/rpm/contrail-nodemgr.spec
 +      -ba {{ ansible_env.HOME }}/{{ packaging.target_dir }}/tools/packaging/common/rpm/contrail-nodemgr.spec \
@@ -32,11 +32,11 @@
 
 @@ -78,7 +81,8 @@
        --define "_builddir {{ ansible_env.HOME }}/{{ packaging.target_dir }}/BUILD" \
-       --define "_buildTag {{ zuul.change }}.{{ zuul.patchset }}" \
+       --define "_buildTag {{ packaging.version.distrib }}" \
        --define "_srcVer {{ packaging.version.upstream }}" \
--      --bb {{ ansible_env.HOME }}/{{ packaging.target_dir }}/tools/packaging/common/rpm/{{ item }}.spec
-+      --bb {{ ansible_env.HOME }}/{{ packaging.target_dir }}/tools/packaging/common/rpm/{{ item }}.spec \
-+      |& tee {{ ansible_env.HOME }}/contrail-build-logs/contrail-logs/{{ item }}_$(date "+%Y%m%d_%H%M%S").log
+-      --bb {{ ansible_env.HOME }}/{{ packaging.target_dir }}/tools/packaging/common/rpm/{{ item.name }}.spec
++      --bb {{ ansible_env.HOME }}/{{ packaging.target_dir }}/tools/packaging/common/rpm/{{ item.name }}.spec \
++      |& tee {{ ansible_env.HOME }}/contrail-build-logs/contrail-logs/{{ item.name }}_$(date "+%Y%m%d_%H%M%S").log
    args:
      chdir: "{{ ansible_env.HOME }}/{{ packaging.target_dir }}/contrail-web-core"
    with_items:

scripts/packaging-build-el-tasks.patch

@@ -1,13 +1,17 @@
 #!/bin/env bash
 set -u
 
-if [ ! -d $HOME/contrail-build-logs ]; then
-   mkdir -p $HOME/contrail-build-logs/contrail-logs
+if [ ! -d ${HOME}/contrail-build-logs ]; then
+   mkdir -p ${HOME}/contrail-build-logs/contrail-logs
 fi
 
-task_dir=$(pwd)/code/contrail-project-config/roles/packaging-build-el/tasks
+task_dir=${1}/contrail-project-config/roles/packaging-build-el/tasks
 patch_dir=$(pwd)/scripts
 
 if ! patch -R -p0 -s -f --dry-run ${task_dir}/main.yaml < ${patch_dir}/packaging-build-el-tasks.patch; then
   patch -p0 ${task_dir}/main.yaml < ${patch_dir}/packaging-build-el-tasks.patch
 fi
+
+if ! patch -R -p0 -s -f --dry-run ${task_dir}/build_package.yaml < ${patch_dir}/build_package.patch; then
+  patch -p0 ${task_dir}/build_package.yaml < ${patch_dir}/build_package.patch
+fi