From 4c3941ff8e299e8ebd5a8822a08fecfb6fd16602 Mon Sep 17 00:00:00 2001
From: Matthew Harmsen <mharmsen@redhat.com>
Date: Wed, 22 Aug 2012 19:49:19 -0700
Subject: [PATCH] Verify symbolic links

* TRAC Ticket #301 - Need to modify init scripts to verify needed
  symlinks in an instance
---
 base/deploy/scripts/operations          |  366 +++++++++++++++++++++++++++++++
 base/deploy/src/scriptlets/pkiparser.py |    2 +-
 base/setup/scripts/functions            |  308 ++++++++++++++++++++++++++
 3 files changed, 675 insertions(+), 1 deletion(-)

diff --git a/base/deploy/scripts/operations b/base/deploy/scripts/operations
index 4f89c1a..ad69ecf 100644
--- a/base/deploy/scripts/operations
+++ b/base/deploy/scripts/operations
@@ -767,6 +767,365 @@ display_instance_status()
     return $rv
 }
 
+fix_symlinks()
+{
+    # declare -p symlinks
+    path="${1}"
+    user="${2}"
+    group="${3}"
+
+    # process key/value pairs in the associative array
+    for key in "${!symlinks[@]}"
+    do
+        symlink="${path}/${key}"
+        if [ -e ${symlink} ]; then
+            if [ -h ${symlink} ]; then
+                target=`readlink ${symlink}`
+                if [ -e ${target} ]; then
+                    # Verify that the target to which the
+                    # symlink points is the expected target!
+                    if [ ${target} == ${symlinks[${key}]} ]; then
+                        # Check that symlink is fully resolvable!
+                        if [ ! `readlink -qe ${symlink}` ]; then
+                            # Issue an ERROR that the target to which the
+                            # symbolic link points is NOT fully resolvable!
+                            echo "ERROR:  '${symlink}' -> '${target}'"
+                            echo "        which is NOT fully resolvable!"
+                            return 1
+                        else
+                            # ALWAYS run 'chown' on '${symlink}'
+                            # NOTE:  Ignore 'chown' errors.
+                            chown -h ${user}:${group} ${symlink}
+                            # echo "SUCCESS:  '${symlink}' -> '${target}'"
+                        fi
+                    else
+                        # Issue an ERROR that the symbolic link does NOT
+                        # point to the expected target and return an error!
+                        echo "ERROR:  '${symlink}' -> '${target}'"
+                        echo "        INSTEAD of '${symlinks[${key}]}'!"
+                        return 1
+                    fi
+                else
+                    # Issue an ERROR that the target to which the
+                    # symbolic link points does NOT exist and return an error!
+                    echo "ERROR:  '${symlink}' -> '${target}'"
+                    echo "        which does NOT exist!"
+                    return 1
+                fi
+            elif [ -f ${symlink} ]; then
+                # Issue a WARNING that the administrator may have replaced
+                # the symbolic link with a file for debugging purposes.
+                echo "WARNING:  '${symlink}' exists but is NOT a symbolic link!"
+            else
+                # Issue a ERROR that the symbolic link has been replaced
+                # by something unusable (such as a directory).
+                echo "ERROR:  '${symlink}' exists but is NOT a symbolic link!"
+            fi
+        else
+            # This symbolic link does not exist; attempt to create it.
+            echo "WARNING:  '${symlink}' does NOT currently exist!"
+            echo "          Attempting to create '${symlink}' . . ."
+            target=${symlinks[${key}]}
+            if [ -e ${target} ]; then
+                # Check that the target is fully resolvable!
+                if [ ! `readlink -qe ${target}` ]; then
+                    # Issue an ERROR that the target to which the
+                    # symbolic link points is NOT fully resolvable!
+                    echo "ERROR:  '${symlink}' will NOT be created because"
+                    echo "        the '${target}' to which it must point"
+                    echo "        is NOT fully resolvable!"
+                    return 1
+                else
+                    # Attempt to create the symlink and chown it.
+                    # NOTE:  Ignore 'chown' errors.
+                    ln -s ${target} ${symlink}
+                    rv=$?
+                    if [ $rv -eq 0 ]; then
+                        chown -h ${user}:${group} ${symlink}
+                        echo "SUCCESS:  Created '${symlink}' -> '${target}'"
+                    else
+                        echo "ERROR:  Failed to create '{symlink}'!"
+                        return 1
+                    fi
+                fi
+            else
+                # Issue an ERROR that NEITHER the symbolic link NOR the
+                # target to which the symbolic link points exist and
+                # return an error!
+                echo "ERROR:  Neither '${symlink}' nor '${target}' exist!"
+                return 1
+            fi
+        fi
+    done
+
+    return 0
+}
+
+# Detect and correct any missing or incorrect symlinks.
+#
+#     Use the following command to locate PKI 'instance' symlinks:
+#
+#        find ${PKI_INSTANCE_PATH} -type l | sort | xargs file
+#
+verify_symlinks()
+{
+    # declare associative arrays
+    declare -A base_symlinks
+    declare -A root_symlinks
+    declare -A ca_symlinks
+    declare -A kra_symlinks
+    declare -A ocsp_symlinks
+    declare -A tks_symlinks
+    declare -A common_jar_symlinks
+    declare -A ca_jar_symlinks
+    declare -A kra_jar_symlinks
+    declare -A ocsp_jar_symlinks
+    declare -A tks_jar_symlinks
+    declare -A systemd_symlinks
+
+    # Dogtag 10 Conditional Variables
+    if [ ${ARCHITECTURE} == "x86_64" ]; then
+        jni_dir="/usr/lib64/java"
+    else
+        jni_dir="/usr/lib/java"
+    fi
+
+    # Dogtag 10 Symbolic Link Target Variables
+    java_dir="/usr/share/java"
+    pki_java_dir="${java_dir}/pki"
+    pki_systemd_service="pki-${PKI_WEB_SERVER_TYPE}d@.service"
+    resteasy_java_dir="${java_dir}/resteasy"
+    sbin_dir="/usr/sbin"
+    systemd_dir="/lib/systemd/system"
+
+    # Dogtag 10 Symbolic Link Variables
+    pki_ca_dir="${PKI_INSTANCE_PATH}/ca"
+    pki_kra_dir="${PKI_INSTANCE_PATH}/kra"
+    pki_ocsp_dir="${PKI_INSTANCE_PATH}/ocsp"
+    pki_tks_dir="${PKI_INSTANCE_PATH}/tks"
+    pki_common_jar_dir="${PKI_INSTANCE_PATH}/common/lib"
+    pki_ca_jar_dir="${pki_common_jar_dir}"
+    pki_kra_jar_dir="${pki_common_jar_dir}"
+    pki_ocsp_jar_dir="${pki_common_jar_dir}"
+    pki_tks_jar_dir="${pki_common_jar_dir}"
+    pki_log_dir="/var/log/pki/${PKI_INSTANCE_ID}"
+    pki_cfg_dir="/etc/pki/${PKI_INSTANCE_ID}"
+    pki_registry_dir="/etc/sysconfig/pki/${PKI_WEB_SERVER_TYPE}/${PKI_INSTANCE_ID}"
+    pki_systemd_dir="/etc/systemd/system/pki-tomcatd.target.wants"
+    pki_systemd_link="pki-${PKI_WEB_SERVER_TYPE}d@${PKI_INSTANCE_ID}.service"
+
+    # '${PKI_INSTANCE_PATH}' symlinks
+    base_symlinks=(
+        [alias]=${pki_cfg_dir}/alias
+        [bin]=/usr/share/tomcat/bin
+        [conf]=${pki_cfg_dir}
+        [lib]=/usr/share/tomcat/lib
+        [logs]=${pki_log_dir})
+
+    # '${PKI_INSTANCE_PATH}' symlinks (root:root ownership)
+    root_symlinks[${PKI_INSTANCE_ID}]=${sbin_dir}/tomcat-sysd
+
+    # '${pki_ca_dir}' symlinks
+    ca_symlinks=([alias]=${PKI_INSTANCE_PATH}/alias
+                 [conf]=${pki_cfg_dir}/ca
+                 [logs]=${pki_log_dir}/ca
+                 [registry]=${pki_registry_dir}
+                 [webapps]=${PKI_INSTANCE_PATH}/webapps)
+
+    # '${pki_ca_jar_dir}' symlinks
+    ca_jar_symlinks[pki-ca.jar]=${pki_java_dir}/pki-ca.jar
+
+    # '${pki_kra_dir}' symlinks
+    kra_symlinks=([alias]=${PKI_INSTANCE_PATH}/alias
+                  [conf]=${pki_cfg_dir}/kra
+                  [logs]=${pki_log_dir}/kra
+                  [registry]=${pki_registry_dir}
+                  [webapps]=${PKI_INSTANCE_PATH}/webapps)
+
+    # '${pki_kra_jar_dir}' symlinks
+    kra_jar_symlinks[pki-kra.jar]=${pki_java_dir}/pki-kra.jar
+
+    # '${pki_ocsp_dir}' symlinks
+    ocsp_symlinks=([alias]=${PKI_INSTANCE_PATH}/alias
+                   [conf]=${pki_cfg_dir}/ocsp
+                   [logs]=${pki_log_dir}/ocsp
+                   [registry]=${pki_registry_dir}
+                   [webapps]=${PKI_INSTANCE_PATH}/webapps)
+
+    # '${pki_ocsp_jar_dir}' symlinks
+    ocsp_jar_symlinks[pki-ocsp.jar]=${pki_java_dir}/pki-ocsp.jar
+
+    # '${pki_tks_dir}' symlinks
+    tks_symlinks=([alias]=${PKI_INSTANCE_PATH}/alias
+                  [conf]=${pki_cfg_dir}/tks
+                  [logs]=${pki_log_dir}/tks
+                  [registry]=${pki_registry_dir}
+                  [webapps]=${PKI_INSTANCE_PATH}/webapps)
+
+    # '${pki_tks_jar_dir}' symlinks
+    tks_jar_symlinks[pki-tks.jar]=${pki_java_dir}/pki-tks.jar
+
+    # '${pki_common_jar_dir}' symlinks
+    common_jar_symlinks=(
+        [apache-commons-codec.jar]=${java_dir}/commons-codec.jar
+        [apache-commons-collections.jar]=${java_dir}/apache-commons-collections.jar
+        [apache-commons-lang.jar]=${java_dir}/apache-commons-lang.jar
+        [apache-commons-logging.jar]=${java_dir}/apache-commons-logging.jar
+        [httpclient.jar]=${java_dir}/httpcomponents/httpclient.jar
+        [httpcore.jar]=${java_dir}/httpcomponents/httpcore.jar
+        [javassist.jar]=${java_dir}/javassist.jar
+        [jaxrs-api.jar]=${resteasy_java_dir}/jaxrs-api.jar
+        [jettison.jar]=${java_dir}/jettison.jar
+        [jss4.jar]=${jni_dir}/jss4.jar
+        [ldapjdk.jar]=${java_dir}/ldapjdk.jar
+        [pki-certsrv.jar]=${pki_java_dir}/pki-certsrv.jar
+        [pki-cms.jar]=${pki_java_dir}/pki-cms.jar
+        [pki-cmsbundle.jar]=${pki_java_dir}/pki-cmsbundle.jar
+        [pki-cmscore.jar]=${pki_java_dir}/pki-cmscore.jar
+        [pki-cmsutil.jar]=${pki_java_dir}/pki-cmsutil.jar
+        [pki-nsutil.jar]=${pki_java_dir}/pki-nsutil.jar
+        [resteasy-atom-provider.jar]=${resteasy_java_dir}/resteasy-atom-provider.jar
+        [resteasy-jaxb-provider.jar]=${resteasy_java_dir}/resteasy-jaxb-provider.jar
+        [resteasy-jaxrs.jar]=${resteasy_java_dir}/resteasy-jaxrs.jar
+        [resteasy-jettison-provider.jar]=${resteasy_java_dir}/resteasy-jettison-provider.jar
+        [scannotation.jar]=${java_dir}/scannotation.jar
+        [symkey.jar]=${jni_dir}/symkey.jar
+        [tomcatjss.jar]=${java_dir}/tomcat7jss.jar
+        [velocity.jar]=${java_dir}/velocity.jar
+        [xerces-j2.jar]=${java_dir}/xerces-j2.jar
+        [xml-commons-apis.jar]=${java_dir}/xml-commons-apis.jar
+        [xml-commons-resolver.jar]=${java_dir}/xml-commons-resolver.jar)
+
+    # '${pki_systemd_dir}' symlinks
+    systemd_symlinks[${pki_systemd_link}]=${systemd_dir}/${pki_systemd_service}
+
+    # Detect and correct 'Tomcat' symbolic links
+    #
+    #     (1) convert the specified associative array into a string
+    #     (2) create a new global 'symlinks' associative array from this
+    #         specified string which will be used by the "fix_symlinks()"
+    #         subroutine
+    #     (3) call "fix_symlinks()" with the appropriate arguments to
+    #         detect and correct this specified associative array;
+    #         "fix_symlinks()" returns 0 on success and 1 on failure
+    #
+    if [ ${PKI_WEB_SERVER_TYPE} == 'tomcat' ]; then
+        # Detect and correct 'base_symlinks'
+        base_symlinks_string=$(declare -p base_symlinks)
+        eval "declare -A symlinks=${base_symlinks_string#*=}"
+        fix_symlinks ${PKI_INSTANCE_PATH} ${PKI_USER} ${PKI_GROUP}
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+
+        # Detect and correct 'root_symlinks'
+        root_symlinks_string=$(declare -p root_symlinks)
+        eval "declare -A symlinks=${root_symlinks_string#*=}"
+        fix_symlinks ${PKI_INSTANCE_PATH} "root" "root"
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+
+        if [ -e ${pki_ca_dir} ]; then
+            # Detect and correct 'ca_symlinks'
+            ca_symlinks_string=$(declare -p ca_symlinks)
+            eval "declare -A symlinks=${ca_symlinks_string#*=}"
+            fix_symlinks ${pki_ca_dir} ${PKI_USER} ${PKI_GROUP} 
+            rv=$?
+            if [ $rv -ne 0 ]; then
+                return $rv
+            fi
+            # Detect and correct 'ca_jar_symlinks'
+            ca_jar_symlinks_string=$(declare -p ca_jar_symlinks)
+            eval "declare -A symlinks=${ca_jar_symlinks_string#*=}"
+            fix_symlinks ${pki_ca_jar_dir} ${PKI_USER} ${PKI_GROUP} 
+            rv=$?
+            if [ $rv -ne 0 ]; then
+                return $rv
+            fi
+        fi
+
+        if [ -e ${pki_kra_dir} ]; then
+            # Detect and correct 'kra_symlinks'
+            kra_symlinks_string=$(declare -p kra_symlinks)
+            eval "declare -A symlinks=${kra_symlinks_string#*=}"
+            fix_symlinks ${pki_kra_dir} ${PKI_USER} ${PKI_GROUP} 
+            rv=$?
+            if [ $rv -ne 0 ]; then
+                return $rv
+            fi
+            # Detect and correct 'kra_jar_symlinks'
+            kra_jar_symlinks_string=$(declare -p kra_jar_symlinks)
+            eval "declare -A symlinks=${kra_jar_symlinks_string#*=}"
+            fix_symlinks ${pki_kra_jar_dir} ${PKI_USER} ${PKI_GROUP} 
+            rv=$?
+            if [ $rv -ne 0 ]; then
+                return $rv
+            fi
+        fi
+
+        if [ -e ${pki_ocsp_dir} ]; then
+            # Detect and correct 'ocsp_symlinks'
+            ocsp_symlinks_string=$(declare -p ocsp_symlinks)
+            eval "declare -A symlinks=${ocsp_symlinks_string#*=}"
+            fix_symlinks ${pki_ocsp_dir} ${PKI_USER} ${PKI_GROUP} 
+            rv=$?
+            if [ $rv -ne 0 ]; then
+                return $rv
+            fi
+            # Detect and correct 'ocsp_jar_symlinks'
+            ocsp_jar_symlinks_string=$(declare -p ocsp_jar_symlinks)
+            eval "declare -A symlinks=${ocsp_jar_symlinks_string#*=}"
+            fix_symlinks ${pki_ocsp_jar_dir} ${PKI_USER} ${PKI_GROUP} 
+            rv=$?
+            if [ $rv -ne 0 ]; then
+                return $rv
+            fi
+        fi
+
+        if [ -e ${pki_tks_dir} ]; then
+            # Detect and correct 'tks_symlinks'
+            tks_symlinks_string=$(declare -p tks_symlinks)
+            eval "declare -A symlinks=${tks_symlinks_string#*=}"
+            fix_symlinks ${pki_tks_dir} ${PKI_USER} ${PKI_GROUP} 
+            rv=$?
+            if [ $rv -ne 0 ]; then
+                return $rv
+            fi
+            # Detect and correct 'tks_jar_symlinks'
+            tks_jar_symlinks_string=$(declare -p tks_jar_symlinks)
+            eval "declare -A symlinks=${tks_jar_symlinks_string#*=}"
+            fix_symlinks ${pki_tks_jar_dir} ${PKI_USER} ${PKI_GROUP} 
+            rv=$?
+            if [ $rv -ne 0 ]; then
+                return $rv
+            fi
+        fi
+
+        # Detect and correct 'common_jar_symlinks'
+        common_jar_symlinks_string=$(declare -p common_jar_symlinks)
+        eval "declare -A symlinks=${common_jar_symlinks_string#*=}"
+        fix_symlinks ${pki_common_jar_dir} ${PKI_USER} ${PKI_GROUP} 
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+
+        # Detect and correct 'systemd_symlinks'
+        systemd_symlinks_string=$(declare -p systemd_symlinks)
+        eval "declare -A symlinks=${systemd_symlinks_string#*=}"
+        fix_symlinks ${pki_systemd_dir} ${PKI_USER} ${PKI_GROUP}
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+    fi
+
+    return 0
+}
+
 start_instance()
 {
     rv=0
@@ -775,6 +1134,13 @@ start_instance()
         rm -f ${RESTART_SERVER}
     fi
 
+    # Verify symbolic links (detecting and correcting them if possible)
+    verify_symlinks
+    rv=$?
+    if [ $rv -ne 0 ] ; then
+        return $rv
+    fi
+
     # Invoke the initscript for this instance
     case $PKI_WEB_SERVER_TYPE in
         tomcat)
diff --git a/base/deploy/src/scriptlets/pkiparser.py b/base/deploy/src/scriptlets/pkiparser.py
index 1fe74e8..4b61284 100644
--- a/base/deploy/src/scriptlets/pkiparser.py
+++ b/base/deploy/src/scriptlets/pkiparser.py
@@ -69,7 +69,7 @@ def process_command_line_arguments(argv):
         mandatory.add_argument('-i',
                                dest='pki_deployed_instance_name',
                                action='store',
-                               nargs=1, metavar='<instance>',
+                               nargs=1, required=True, metavar='<instance>',
                                help='FORMAT:  ${pki_instance_name}'
                                     '[.${pki_admin_domain_name}]')
     # Establish 'Optional' command-line options
diff --git a/base/setup/scripts/functions b/base/setup/scripts/functions
index a4318ef..003434a 100644
--- a/base/setup/scripts/functions
+++ b/base/setup/scripts/functions
@@ -738,6 +738,307 @@ display_instance_status()
     return $rv
 }
 
+fix_symlinks()
+{
+    # declare -p symlinks
+    path="${1}"
+    user="${2}"
+    group="${3}"
+
+    # process key/value pairs in the associative array
+    for key in "${!symlinks[@]}"
+    do
+        symlink="${path}/${key}"
+        if [ -e ${symlink} ]; then
+            if [ -h ${symlink} ]; then
+                target=`readlink ${symlink}`
+                if [ -e ${target} ]; then
+                    # Verify that the target to which the
+                    # symlink points is the expected target!
+                    if [ ${target} == ${symlinks[${key}]} ]; then
+                        # Check that symlink is fully resolvable!
+                        if [ ! `readlink -qe ${symlink}` ]; then
+                            # Issue an ERROR that the target to which the
+                            # symbolic link points is NOT fully resolvable!
+                            echo "ERROR:  '${symlink}' -> '${target}'"
+                            echo "        which is NOT fully resolvable!"
+                            return 1
+                        else
+                            # ALWAYS run 'chown' on '${symlink}'
+                            # NOTE:  Ignore 'chown' errors.
+                            chown -h ${user}:${group} ${symlink}
+                            # echo "SUCCESS:  '${symlink}' -> '${target}'"
+                        fi
+                    else
+                        # Issue an ERROR that the symbolic link does NOT
+                        # point to the expected target and return an error!
+                        echo "ERROR:  '${symlink}' -> '${target}'"
+                        echo "        INSTEAD of '${symlinks[${key}]}'!"
+                        return 1
+                    fi
+                else
+                    # Issue an ERROR that the target to which the
+                    # symbolic link points does NOT exist and return an error!
+                    echo "ERROR:  '${symlink}' -> '${target}'"
+                    echo "        which does NOT exist!"
+                    return 1
+                fi
+            elif [ -f ${symlink} ]; then
+                # Issue a WARNING that the administrator may have replaced
+                # the symbolic link with a file for debugging purposes.
+                echo "WARNING:  '${symlink}' exists but is NOT a symbolic link!"
+            else
+                # Issue a ERROR that the symbolic link has been replaced
+                # by something unusable (such as a directory).
+                echo "ERROR:  '${symlink}' exists but is NOT a symbolic link!"
+            fi
+        else
+            # This symbolic link does not exist; attempt to create it.
+            echo "WARNING:  '${symlink}' does NOT currently exist!"
+            echo "          Attempting to create '${symlink}' . . ."
+            target=${symlinks[${key}]}
+            if [ -e ${target} ]; then
+                # Check that the target is fully resolvable!
+                if [ ! `readlink -qe ${target}` ]; then
+                    # Issue an ERROR that the target to which the
+                    # symbolic link points is NOT fully resolvable!
+                    echo "ERROR:  '${symlink}' will NOT be created because"
+                    echo "        the '${target}' to which it must point"
+                    echo "        is NOT fully resolvable!"
+                    return 1
+                else
+                    # Attempt to create the symlink and chown it.
+                    # NOTE:  Ignore 'chown' errors.
+                    ln -s ${target} ${symlink}
+                    rv=$?
+                    if [ $rv -eq 0 ]; then
+                        chown -h ${user}:${group} ${symlink}
+                        echo "SUCCESS:  Created '${symlink}' -> '${target}'"
+                    else
+                        echo "ERROR:  Failed to create '{symlink}'!"
+                        return 1
+                    fi
+                fi
+            else
+                # Issue an ERROR that NEITHER the symbolic link NOR the
+                # target to which the symbolic link points exist and
+                # return an error!
+                echo "ERROR:  Neither '${symlink}' nor '${target}' exist!"
+                return 1
+            fi
+        fi
+    done
+
+    return 0
+}
+
+# Detect and correct any missing or incorrect symlinks.
+#
+#     Use the following command to locate PKI 'instance' symlinks:
+#
+#        find ${PKI_INSTANCE_PATH} -type l | sort | xargs file
+#
+verify_symlinks()
+{
+    declare -A apache_symlinks
+    declare -A perl_symlinks
+    declare -A base_symlinks
+    declare -A root_symlinks
+    declare -A common_jar_symlinks
+    declare -A webapps_jar_symlinks
+    declare -A systemd_symlinks
+
+    # Dogtag 9 Conditional Variables
+    if [ ${ARCHITECTURE} == "x86_64" ]; then
+        jni_dir="/usr/lib64/java"
+    else
+        jni_dir="/usr/lib/java"
+    fi
+    if   [ ${PKI_SUBSYSTEM_TYPE} == "ca" ]; then
+        pki_systemd_link="pki-cad@${PKI_INSTANCE_ID}.service"
+        pki_systemd_service="pki-cad@.service"
+    elif [ ${PKI_SUBSYSTEM_TYPE} == "kra" ]; then
+        pki_systemd_link="pki-krad@${PKI_INSTANCE_ID}.service"
+        pki_systemd_service="pki-krad@.service"
+    elif [ ${PKI_SUBSYSTEM_TYPE} == "ocsp" ]; then
+        pki_systemd_link="pki-ocspd@${PKI_INSTANCE_ID}.service"
+        pki_systemd_service="pki-ocspd@.service"
+    elif [ ${PKI_SUBSYSTEM_TYPE} == "ra" ]; then
+        pki_systemd_link="pki-rad@${PKI_INSTANCE_ID}.service"
+        pki_systemd_service="pki-rad@.service"
+    elif [ ${PKI_SUBSYSTEM_TYPE} == "tks" ]; then
+        pki_systemd_link="pki-tksd@${PKI_INSTANCE_ID}.service"
+        pki_systemd_service="pki-tksd@.service"
+    elif [ ${PKI_SUBSYSTEM_TYPE} == "tps" ]; then
+        pki_systemd_link="pki-tpsd@${PKI_INSTANCE_ID}.service"
+        pki_systemd_service="pki-tpsd@.service"
+    fi
+
+    # Dogtag 9 Symbolic Link Target Variables
+    perl_dir="/usr/share/pki/${PKI_SUBSYSTEM_TYPE}/lib/perl"
+    java_dir="/usr/share/java"
+    pki_java_dir="${java_dir}/pki"
+    # resteasy_java_dir="${java_dir}/resteasy"
+    pki_run_dir="/var/run/pki/${PKI_SUBSYSTEM_TYPE}"
+    sbin_dir="/usr/sbin"
+    systemd_dir="/lib/systemd/system"
+
+    # Dogtag 9 Symbolic Link Variables
+    pki_docroot_dir="${PKI_INSTANCE_PATH}/docroot"
+    pki_perl_dir="${PKI_INSTANCE_PATH}/lib"
+    pki_common_jar_dir="${PKI_INSTANCE_PATH}/common/lib"
+    pki_log_dir="/var/log/${PKI_INSTANCE_ID}"
+    pki_cfg_dir="/etc/${PKI_INSTANCE_ID}"
+    # pki_registry_dir="/etc/sysconfig/pki/${PKI_SUBSYSTEM_TYPE}/${PKI_INSTANCE_ID}"
+    pki_systemd_dir="/etc/systemd/system/pki-cad.target.wants"
+    pki_webapps_jar_dir="${PKI_INSTANCE_PATH}/webapps/${PKI_SUBSYSTEM_TYPE}/WEB-INF/lib"
+
+    # '${PKI_INSTANCE_PATH}' symlinks
+    apache_symlinks=(
+        [conf]=${pki_cfg_dir}
+        [logs]=${pki_log_dir}
+        [run]=${pki_run_dir})
+
+    base_symlinks=(
+        [conf]=${pki_cfg_dir}
+        [logs]=${pki_log_dir})
+
+    # '${PKI_INSTANCE_PATH}' symlinks (root:root ownership)
+    root_symlinks[${PKI_INSTANCE_ID}]=${sbin_dir}/tomcat6-sysd
+
+    # '${pki_perl_dir}' symlinks
+    perl_symlinks[perl]=${perl_dir}
+
+    # '${pki_common_jar_dir}' symlinks
+    common_jar_symlinks=(
+        [apache-commons-logging.jar]=${java_dir}/apache-commons-logging.jar
+        [jss4.jar]=${jni_dir}/jss4.jar
+        [tomcatjss.jar]=${java_dir}/tomcatjss.jar
+        # Dogtag 9 -> Dogtag 10
+        [apache-commons-codec.jar]=${java_dir}/commons-codec.jar)
+
+    # '${pki_webapps_jar_dir}' symlinks
+    webapps_jar_symlinks=(
+        [apache-commons-collections.jar]=${java_dir}/apache-commons-collections.jar
+        [apache-commons-lang.jar]=${java_dir}/apache-commons-lang.jar
+        [ldapjdk.jar]=${java_dir}/ldapjdk.jar
+        # [osutil.jar]=${jni_dir}/osutil.jar
+        [${PKI_INSTANCE_ID}.jar]=${pki_java_dir}/${PKI_INSTANCE_ID}.jar
+        [pki-certsrv.jar]=${pki_java_dir}/pki-certsrv.jar
+        [pki-cms.jar]=${pki_java_dir}/pki-cms.jar
+        [pki-cmsbundle.jar]=${pki_java_dir}/pki-cmsbundle.jar
+        [pki-cmscore.jar]=${pki_java_dir}/pki-cmscore.jar
+        [pki-cmsutil.jar]=${pki_java_dir}/pki-cmsutil.jar
+        [pki-nsutil.jar]=${pki_java_dir}/pki-nsutil.jar
+        [symkey.jar]=${jni_dir}/symkey.jar
+        [velocity.jar]=${java_dir}/velocity.jar
+        [xerces-j2.jar]=${java_dir}/xerces-j2.jar
+        [xml-commons-apis.jar]=${java_dir}/xml-commons-apis.jar
+        [xml-commons-resolver.jar]=${java_dir}/xml-commons-resolver.jar)
+
+    # '${pki_systemd_dir}' symlinks
+    systemd_symlinks[${pki_systemd_link}]=${systemd_dir}/${pki_systemd_service}
+
+    # Detect and correct PKI subsystem 'instance' symbolic links
+    #
+    #     (1) convert the specified associative array into a string
+    #     (2) create a new global 'symlinks' associative array from this
+    #         specified string which will be used by the "fix_symlinks()"
+    #         subroutine
+    #     (3) call "fix_symlinks()" with the appropriate arguments to
+    #         detect and correct this specified associative array;
+    #         "fix_symlinks()" returns 0 on success and 1 on failure
+    #
+    if   [ "${PKI_SUBSYSTEM_TYPE}" == "ra"   ]  ||
+         [ "${PKI_SUBSYSTEM_TYPE}" == "tps"  ]
+    then
+        # Detect and correct 'apache_symlinks'
+        apache_symlinks_string=$(declare -p apache_symlinks)
+        eval "declare -A symlinks=${apache_symlinks_string#*=}"
+        fix_symlinks ${PKI_INSTANCE_PATH} ${PKI_USER} ${PKI_GROUP}
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+
+        # Detect and correct 'perl_symlinks'
+        perl_symlinks_string=$(declare -p perl_symlinks)
+        eval "declare -A symlinks=${perl_symlinks_string#*=}"
+        fix_symlinks ${pki_perl_dir} ${PKI_USER} ${PKI_GROUP}
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+
+        if [ "${PKI_SUBSYSTEM_TYPE}" == "tps"  ]; then
+            # ALWAYS recreate this "convenience" link and 'chown' it
+            # NOTE:  Ignore 'chown' errors.
+            cd ${pki_docroot_dir} ;
+            ln -s tokendb tus ;
+            rv=$?
+            if [ $rv -eq 0 ]; then
+                chown -h ${PKI_USER}:${PKI_GROUP} tus
+                # echo "SUCCESS:  Created 'tus' -> 'tokendb'"
+            else
+                echo "ERROR:  Failed to create 'tus' -> 'tokendb' convenience"
+                echo "        symbolic link for '${PKI_INSTANCE_ID}'!"
+                return 1
+            fi
+        fi
+    elif [ "${PKI_SUBSYSTEM_TYPE}" == "ca"   ]  ||
+         [ "${PKI_SUBSYSTEM_TYPE}" == "kra"  ]  ||
+         [ "${PKI_SUBSYSTEM_TYPE}" == "ocsp" ]  ||
+         [ "${PKI_SUBSYSTEM_TYPE}" == "tks"  ]
+    then
+        # Detect and correct 'base_symlinks'
+        base_symlinks_string=$(declare -p base_symlinks)
+        eval "declare -A symlinks=${base_symlinks_string#*=}"
+        fix_symlinks ${PKI_INSTANCE_PATH} ${PKI_USER} ${PKI_GROUP}
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+
+        # Detect and correct 'root_symlinks'
+        root_symlinks_string=$(declare -p root_symlinks)
+        eval "declare -A symlinks=${root_symlinks_string#*=}"
+        fix_symlinks ${PKI_INSTANCE_PATH} "root" "root"
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+
+        # Detect and correct 'common_jar_symlinks'
+        common_jar_symlinks_string=$(declare -p common_jar_symlinks)
+        eval "declare -A symlinks=${common_jar_symlinks_string#*=}"
+        fix_symlinks ${pki_common_jar_dir} ${PKI_USER} ${PKI_GROUP}
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+
+        # Detect and correct 'webapps_jar_symlinks'
+        webapps_jar_symlinks_string=$(declare -p webapps_jar_symlinks)
+        eval "declare -A symlinks=${webapps_jar_symlinks_string#*=}"
+        fix_symlinks ${pki_webapps_jar_dir} ${PKI_USER} ${PKI_GROUP}
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+
+        # Detect and correct 'systemd_symlinks'
+        systemd_symlinks_string=$(declare -p systemd_symlinks)
+        eval "declare -A symlinks=${systemd_symlinks_string#*=}"
+        fix_symlinks ${pki_systemd_dir} ${PKI_USER} ${PKI_GROUP}
+        rv=$?
+        if [ $rv -ne 0 ]; then
+            return $rv
+        fi
+    fi
+
+    return 0
+}
+
 start_instance()
 {
     rv=0
@@ -746,6 +1047,13 @@ start_instance()
         rm -f ${RESTART_SERVER}
     fi
 
+    # Verify symbolic links (detecting and correcting them if possible)
+    verify_symlinks
+    rv=$?
+    if [ $rv -ne 0 ] ; then
+        return $rv
+    fi
+
     # Invoke the initscript for this instance
     case $PKI_SUBSYSTEM_TYPE in
         ca|kra|ocsp|tks)
-- 
1.7.10.4

