>From 206f436e24c4bc212480abb9f83c4bd27bda815c Mon Sep 17 00:00:00 2001
From: Abhishek Koneru <akoneru@redhat.com>
Date: Wed, 20 Nov 2013 01:01:52 -0500
Subject: [PATCH] Updated the tests README with the CI server setup info.

Updated the README with all the information required for setting
up a CI test framework for Dogtag. This patch also addresses the
ticket 726 which to add an email notification for the CI build/test
regression cases.

Ticket #726
---
 tests/dogtag/README                   | 147 +++++++++++++++++++++++++++++-----
 tests/dogtag/util/examine_results.sh  |  37 +++++++++
 tests/dogtag/util/find_regressions.py |  52 ++++++++++++
 3 files changed, 216 insertions(+), 20 deletions(-)
 create mode 100755 tests/dogtag/util/examine_results.sh
 create mode 100755 tests/dogtag/util/find_regressions.py

diff --git a/tests/dogtag/README b/tests/dogtag/README
index 33da9828a90769fcfc514ac1ec7ddee5e57ee188..0d6715d03bed82fdc7a5e20f5f454efadc14a749 100644
--- a/tests/dogtag/README
+++ b/tests/dogtag/README
@@ -108,7 +108,7 @@
         This command builds the rpms, copies them to the REPOSITORY_LOCATION specified in the file
         and creates it as a repository.
         If the location specified is hosted on a httpd server, this repository
-        URL can be in the job xml.
+        URL can be used in the job xml as a repo URL.
 
         If a remote host name is provided then the repos are scp'ed at the
         location provided on the remote host using the credentials provided.
@@ -166,42 +166,149 @@
 
 2. Setting up a Jenkins server for continuous integration testing
 
-  -- Follow the steps in section 1.2.1 to setup a beaker-client on the
-     system.
+  Note: Follow the steps in section 1.2.1 to setup a beaker-client on the
+        system.
 
-  -- Install Jenkins. (It is better to configure the Jenkins server to use a
-                       port other than 8080)
+  2.1 Setting up a  Jenkins server.
 
-  -- Create a New Project - ex: Dogtag-pki
+  2.1.1 Installing the Jenkins (and plugins)
 
-  -- Configuring the project (Click on "Configure" in the left panel):
+      The instructions for installing a jenkins server can be found at:
+      https://wiki.jenkins-ci.org/display/JENKINS/Installing+Jenkins.
+      (It is better to configure the Jenkins server to listen on a
+       port other than 8080 to not raise port conflicts. The jenkins
+       configuraion file is at /etc/sysconfig/jenkins. A server restart is
+       necessary for any changes to this file to be effective. )
 
-     * Setting up the workspace:
+      The Jenkins UI can be accessed by the URL:
+        http://<hostname>:<port>/, where the hostname is the hostname of the
+      machine where Jenkins is installed and port is the port on which the
+      server is listening.
+
+      The CI test framework requires a few additional plugins to be installed
+      for the Jenkins server.
+
+      -- Goto Jenkins -> Manage Jenkins -> Manage Plugins page on the UI.
+         Click on the "Available" tab.
+         Search and install the following plugins:
+             * GitHub plugin
+             * Jenkins Beaker builder plugin
+             * Jenkins Email Extension Plugin
+
+  2.1.2 Configuring Jenkins setup
+      
+      The following configurations are required for the setup to be 
+      complete:
+      Goto Jenkins -> Manage Jenkins -> Configure System.
+
+      Update the "System Admin e-mail address" in "Jenkins Location" section.
+ 
+      Enter the details of the beaker server in the "Beaker Builder" section.
+      The server's certificate needs to be stored in a Java keystore if the
+      connection uses https protocol. The details for fixing this issue can be
+      found in the plugin documentation at:
+      https://wiki.jenkins-ci.org/display/JENKINS/Beaker+Builder+Plugin.
+
+      The "Email Notification" section needs to be completed with the details 
+      of the SMTP server and its authentication details (Click on the Advanced
+      button).
+         
+  2.2  Create a New Project - ex: Dogtag-pki
+
+  2.2.1 Configuring the project (Click on "Configure" in the left panel):
+
+      Setting up the workspace:
+    
+      Set the path to the custom workspace in the Advanced Project Options
+      section to "<Any path on the system>/pki"
+      Click on Advanced -> Select Use custom workspace -> Enter the path.
+      The build proces of Dogtag requires the top source directory to be 
+      named as pki. Hence it is required to setup a the workspace path.
+      To use the default Jenkins workspace, set the path as
+      /var/lib/jenkins/workspace/pki.
 
        i. For continuous integration, the project can be configured to clone
           the source from the repository (git://git.fedorahosted.org/git/pki.git).
-          To use the Git SCM, install the Jenkins-Git plugin.
-          Goto Jenkins -> Manage Jenkins -> Manage Plugins to find and install
-          the plugin. The Git option can be found in the SCM section.
+          The Git option can be found in the SCM section. 
 
        ii. For using the working project directory as the source to build the
-           rpms - Select None in the SCM section and set the path to the
-           project root (pki) as custom workspace in the Advanced Project Options section.
-           Click on Advanced -> Select Use custom workspace -> Enter the path.
+           rpms - Select None in the SCM section and clone the source in the
+           above mentioned location.
 
-   -- Building the rpms and running the tests.
+   2.2.2 Building the rpms and running the tests.
 
       In the "Build" section, select "Add build step" -> "Execute Shell"
       Paste the following code in the Command text box.
 
       cd $WORKSPACE/scripts; ./compose_pki_core_packages [--createrepo=<repository_file_absolute_path>] hybrid_rpms
       echo "Compose the test package and submit the beaker task and job"
-      cd $WORKSPACE/scripts; ./compose_pki_test_package <UserID> <Job_XML_config_file_absolute_path> --runtests
+      cd $WORKSPACE/scripts; ./compose_pki_test_package dogtagCI --submit-tests
 
-      This will build the rpms, create the repo(if specified), compose
-      the test rpm and submit the tests to the beaker.
-      (The description to the commands used and setting up the beaker-client
-       are specified in the previous sections of this document)
+      This shell script builds the pki core rpms, copies them to a repository,
+      creates the test rpms using the unique_identifier dogtagCI and submits
+      it to the beaker server.
+
+      Then, select "Add build step" -> "Execute Beaker Task".
+      Select the option "Specify Job XML".
+      Copy the text in the pki/tests/dogtag/beakerjob.dogtag.xml.template file
+      into the test area.
+      Replace the string PKI_TEST_USER_ID in the task name to dogtagCI.
+      Enter values for the distro required in the distroRequires element of the
+      XML. 
+      Add the repository URLs in the repos element as follows:
+      <repo name="repo1" url="<URL_to_access_the_repos>"/>
+      <repo name="repo2" url="<URL_to_access_the_repos>"/>
+      
+      Again, select "Add build step" -> "Execute shell"
+      Paste the following code in the text box:
+
+      echo "Check if there are any regressions in the current beaker test run."
+      cd $WORKSPACE/tests/dogtag/util
+      ./examine_results.sh ${BEAKER_JOB_ID}
+
+      This shell script examines the results of the curent and the previous run
+      and send an email as per the triggers, only if any regressions are found.
+
+   2.2.3 Setting up the mail client
+
+      For setting up the post build email notification, select 
+      "Add post-build action" -> "Email Notification". This can be used to 
+      specify the email recipients and configuring when and to whom the email
+      has to be sent.
+
+      Configuring only this action will send emails for any result of the
+      build process to all the recipients. (Even if the build passes 
+      successfully always.)
+
+      To get an email only when there are any regressions, add another post
+      build action, "Add post-build action" -> "Edit Email Notification".
+      This action provides additional options to configure the email itself.
+      
+     Click on "Advanced" button and copy the following code into the 
+     Pre-send script text box :
+
+     def fileName = new FilePath(build.getWorkspace(), "/beaker/regressions.txt");
+     if (fileName.exists) {
+         def file = new File(fileName.toString())
+         msg.setText(file.getText());
+     }else{
+         cancel=true;
+     }
+
+     This Groovy script looks for a file regressions.txt, created by the 
+     examine_results script mentioned shown in the previous section. The 
+     file contains the names of the tests that passed previously 
+     but failed now. A mail will also be generated if there are tests that 
+     passed now and were failing before.
+
+    This helps reduce spamming of emails when running the continuous 
+    integration testng framework.
+ 
+    The "Add Trigger" selection box will be useful to add a trigger to 
+    configure for which  result of the build an email has to be sent
+    to the recipients.
+
+   Additional Information:
 
    -- The Build Triggers section provides options for specifying the time at
       which a new build can be triggered - periodically or on a git commit.
diff --git a/tests/dogtag/util/examine_results.sh b/tests/dogtag/util/examine_results.sh
new file mode 100755
index 0000000000000000000000000000000000000000..2a3657b51b51b82db1e95db4b8338f9201e3af4a
--- /dev/null
+++ b/tests/dogtag/util/examine_results.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+CURRENT_JOB_ID=$1
+
+OLD_JOB_ID=""
+
+BEAKER_DIR="`cd ../../../; pwd`/beaker"
+
+if [ ! -d $BEAKER_DIR ] ; then
+    mkdir $BEAKER_DIR
+fi
+
+rm -rf $BEAKER_DIR/regressions.txt
+
+if [ -f $BEAKER_DIR/last_beaker_run ] ; then
+    OLD_JOB_ID=`cat $BEAKER_DIR/last_beaker_run`
+else
+    echo $CURRENT_JOB_ID > $BEAKER_DIR/last_beaker_run
+    exit 0
+fi
+
+bkr job-results --prettyxml "J:$CURRENT_JOB_ID" > $BEAKER_DIR/logsxml.new
+
+bkr job-results --prettyxml "J:$OLD_JOB_ID" > $BEAKER_DIR/logsxml.old
+
+rm -rf $BEAKER_DIR/last_beaker_run
+
+echo $CURRENT_JOB_ID > $BEAKER_DIR/last_beaker_run
+
+python find_regressions.py $BEAKER_DIR/logsxml.old $BEAKER_DIR/logsxml.new
+
+if [ $? -gt 0 ] ; then
+   echo "No Regressions Found."
+   exit 0
+fi
+
+mv regressions.txt $BEAKER_DIR/
diff --git a/tests/dogtag/util/find_regressions.py b/tests/dogtag/util/find_regressions.py
new file mode 100755
index 0000000000000000000000000000000000000000..29b64e3ea17055765e8f6fc79c52dd64ca8a570c
--- /dev/null
+++ b/tests/dogtag/util/find_regressions.py
@@ -0,0 +1,52 @@
+#!/usr/bin/python
+
+import sys
+from lxml import etree
+
+### Usage python find_regressions.py <old_log.xml> <new_log.xml>
+
+doc = etree.parse(str(sys.argv[1]))
+
+tasks = doc.xpath("/job/recipeSet/recipe/task")
+task_results = dict()
+result_status = None
+
+for task in tasks:
+    task_name = task.attrib['name']
+    results = task.xpath("results/result")
+    if results is not None:
+        result_status = dict()
+    for result in results:
+        result_status[result.attrib['path']] = result.attrib['result']
+
+    task_results[task_name] = result_status
+    result_status = None
+
+## Compare the new ones
+
+doc = etree.parse(str(sys.argv[2]))
+
+tasks = doc.xpath("/job/recipeSet/recipe/task")
+
+pass_to_fail = []
+fail_to_pass = []
+
+for task in tasks:
+    task_name = task.attrib['name']
+    curr_results = task.xpath("results/result")
+    old_results = task_results[task_name]
+    if curr_results is not None:
+        for result in curr_results:
+            if result.attrib['result'] != old_results[result.attrib['path']]:
+                 if result.attrib['result'] == "Fail":
+                     fail_to_pass.append(result.attrib['path'])
+                 else:
+                     pass_to_fail.append(result.attrib['path'])
+if len(pass_to_fail) == 0 and len(fail_to_pass) == 0:
+    sys.exit(1)
+
+with open("regressions.txt", "w") as f:
+    if len(pass_to_fail) > 0:
+        f.write("Regressions: " + str(pass_to_fail) + "\n")
+    if len(fail_to_pass) > 0:
+        f.write("Tests that pass with the new changes: " + str(fail_to_pass) + "\n")
-- 
1.8.3.1

