# Avaya CMS

# Troubleshooting

# Avaya CMS - NFS Not Mounted local disk full

CMS Disk getting full reason was daily backups where being stored loally as NFS mount did not automatically mount after a reboot.

```shell
[root@lab-cms /]# df -kh
Filesystem      Size  Used Avail Use% Mounted on
/dev/sda2       9.8G  9.8G     0 100% /
tmpfs            32G     0   32G   0% /dev/shm
/dev/sda1       546M   38M  480M   8% /boot
/dev/sda3       9.8G  1.8G  7.5G  19% /cms
/dev/sda7        32G  6.1G   24G  21% /export/home
/dev/sda10       12G  759M   11G   7% /opt
/dev/sda6        87G   13G   70G  15% /storage
/dev/sda9        16G   45M   15G   1% /tmp
/dev/sda8        26G  297M   24G   2% /var
See output here, there is no nfs drive mounted.

I have helped to remount the drive like this:

[root@lab-cms /]# mount -a
[root@lab-cms /]# df -kh
Filesystem            Size  Used Avail Use% Mounted on
/dev/sda2       9.8G  9.8G     0 100% /
tmpfs                  32G     0   32G   0% /dev/shm
/dev/sda1             546M   38M  480M   8% /boot
/dev/sda3             9.8G  1.8G  7.5G  19% /cms
/dev/sda7              32G  6.1G   24G  21% /export/home
/dev/sda10             12G  759M   11G   7% /opt
/dev/sda6              87G   13G   70G  15% /storage
/dev/sda9              16G   45M   15G   1% /tmp
/dev/sda8              26G  8.6G   16G  36% /var
10.10.10.122:/CMS-Share
                       50G   35G   16G  70% /nfsbu

As the issue is marked as SBI,  I have helped to move the backup files which is around 8GB from local partition to nfs location. And the whole procedure is not service impacting.

Now the partition issue is solved

[root@lab-cms var]# df -kh
Filesystem            Size  Used Avail Use% Mounted on
/dev/sda2             9.8G  1.5G  7.8G  16% /
tmpfs                  32G     0   32G   0% /dev/shm
/dev/sda1             546M   38M  480M   8% /boot
/dev/sda3             9.8G  1.8G  7.5G  19% /cms
/dev/sda7              32G  6.1G   24G  21% /export/home
/dev/sda10             12G  759M   11G   7% /opt
/dev/sda6              87G   13G   70G  15% /storage
/dev/sda9              16G   45M   15G   1% /tmp
/dev/sda8              26G  297M   24G   2% /var
10.10.10.122:/CMS-Share
                       50G   43G  7.2G  86% /nfsbu


```

# Avaya CMS - CMS Supervisor status link icon

[![cms sup status link.PNG](https://wiki.tinod.net/uploads/images/gallery/2023-04/scaled-1680-/97e1DyRSawrhzSqp-cms-sup-status-link.PNG)](https://wiki.tinod.net/uploads/images/gallery/2023-04/97e1DyRSawrhzSqp-cms-sup-status-link.PNG)

# Avaya CMS - Healthcheck commands

who -q -- check who's logged on  
rpm -qa cms -- View CMS Version  
df -h -- check drive space (GB)  
df -i -- check drive space (b)  
iostat -- monitor input/output device average transfer rates (for more info https://linux.die.net/man/1/iostat)  
cronetab -l -- list cronejobs  
/cms/aom/bin/active\_alarms  
cat /cms/aom/data/db/alarm.lst  
cat /cms/aom/data/log/alarm.log  
tail /cms/pbx/acd1/spi.err  
tail /cms/pbx/acd1/xln.log  
tail /cms/dc/archive/arch.log  
tail /cms/dc/harchive/harch.log  
tail /cms/install/logdir/backup.log  
tail /cms/maint/backup/back.log  
/cms/aom/bin/active\_alarms  
cat /cms/aom/data/db/alarm.1st  
cat /cms/aom/data/log/alarm.log  
cat /var/elog/elog - view elog (Prognosis if installed)  
cat /var/log/messages  
ls /usr/lib/HA/ -- shows if CMS is primary or seconday  
cat /export/home/pserv/HAcms/Log -- Heartbeats and sync log  
/export/home/pserv/HAcms/HAheartbeat.log -- check Hearbeats between servers  
/export/home/c1as/xcpr/xcpr\_status

# Avaya CMS - Find zombie process

pps auxww

Z = Zombie Process

D = uninterruptible sleep.

```bash
$ ps auxww | grep "Z"
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root       935  0.0  0.0      0     0 ?        Z    17:40   0:00 [ech_copy.sh] <defunct>
cms       6569  0.0  0.0 103324   868 pts/4    S+   17:50   0:00 grep Z
$ find ech_copy.sh
find: `ech_copy.sh': No such file or directory
$ locate ech_copy.sh
/export/home/pserv/ech/ech_copy.sh
$ cat /export/home/pserv/ech/ech_copy.sh
#!/bin/ksh
 ############################################################################
 #                                                                          #
 #       Copyright (c) 1997 - 2017 by Avaya Inc. All rights reserved.       #
 #                                                                          #
 ############################################################################
#
# ech_copy.sh
#

# Load common functions and environment settings
BASEDIR=$(dirname $0)
. ${BASEDIR}/functions

# Set local variables
#HOME=/export/home/pserv/ech
. ${PKGHOME}/ech.conf         # ECH configuration variables file.
# . ${PKGHOME}/cvx_ftp.funcs    # declare functions

logit "============================================="
logit "Calling ech_copy"

# See if a timestamp file name is requested
if [[ "$TS_NAME" == "yes" ]]
then
  ts=`date +$TS_FORMAT`
  chr_name=`echo $2 | cut -f1 -d.`
  new_name="${chr_name}.${ts}"
  ${PKGHOME}/ech_copy $1 $new_name
else
  ${PKGHOME}/ech_copy $1 $2
fi
```

```bash
$ ps auxww | grep "D"
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root      4999  0.0  0.0   6872   796 ?        Ss   Mar11   2:48 /export/home/pserv/rt_socket/rt_socket1 -w29 -w 10 -a 5000 -E ==EOD== new_rtsocket 7000 1
root      5191  0.0  0.0   7528   956 ?        Ss   Mar11   2:25 /export/home/pserv/rt_socket/rt_socket2 -w29 -w 10 -a 10000 -E ==EOD== new_rtsocket 7002 1
root      5324  0.0  0.0   6872   984 ?        Ss   Mar11  14:07 /export/home/pserv/rt_socket/rt_socket3 -w29 -w 10 -a 5000 -E ==EOD== new_rtsocket 7001 1
root      5455  0.0  0.0   7528   880 ?        Ss   Mar11   1:29 /export/home/pserv/rt_socket/rt_socket4 -w29 -w 10 -a 10000 -E ==EOD== new_rtsocket 7003 1
root      5543  0.0  0.0   7528   936 ?        Ss   Mar11   2:11 /export/home/pserv/rt_socket/rt_socket5 -w29 -w 10 -a 10000 -E ==EOD== new_rtsocket 7004 1
root      5677  0.0  0.0   6872   756 ?        Ss   Mar11   0:19 /export/home/pserv/rt_socket/rt_socket6 -w29 -a 5000 -E ==EOD== -M180 -w90 COSVMDEV01 6996 1
cms       7005  0.0  0.0 103324   876 pts/4    S+   17:51   0:00 grep D

```

# Avaya CMS - Autosync

<div id="bkmrk-"><div id="bkmrk--1"><div id="bkmrk--2"><div id="bkmrk--3"><div class="splitToolbarButton toggled" id="bkmrk--4" role="radiogroup"></div></div></div></div></div><div id="bkmrk--5"><div id="bkmrk--6"></div></div><div id="bkmrk--7"></div><div id="bkmrk--8"><div id="bkmrk--9"></div></div>Click on Link below to display PDF.

<div id="bkmrk--10"><div id="bkmrk--11"><div id="bkmrk--12" tabindex="0"><div class="pdfViewer" id="bkmrk--13" lang="en-US" style="--scale-factor: 1.6666666666666665;"><div aria-label="Page 2" class="page" data-loaded="true" data-page-number="2" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">[How Autosync Works.pdf](https://wiki.tinod.net/attachments/1)</div><div aria-label="Page 3" class="page" data-page-number="3" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 4" class="page" data-page-number="4" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 5" class="page" data-page-number="5" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 6" class="page" data-page-number="6" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 7" class="page" data-page-number="7" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 8" class="page" data-page-number="8" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 9" class="page" data-page-number="9" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 10" class="page" data-page-number="10" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 11" class="page" data-page-number="11" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 12" class="page" data-page-number="12" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 13" class="page" data-page-number="13" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 14" class="page" data-page-number="14" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 15" class="page" data-page-number="15" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 16" class="page" data-page-number="16" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 17" class="page" data-page-number="17" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 18" class="page" data-page-number="18" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 19" class="page" data-page-number="19" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 20" class="page" data-page-number="20" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 21" class="page" data-page-number="21" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 22" class="page" data-page-number="22" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 23" class="page" data-page-number="23" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div><div aria-label="Page 24" class="page" data-page-number="24" role="region" style="width: calc(var(--scale-factor) * 612px); height: calc(var(--scale-factor) * 792px);">  
</div></div></div></div><div id="bkmrk--14"></div></div>

# ECH Data

cd /cms/ech\_data/archive  
ls -lt | more to identify dates to re upload

2024144.bz2  
cp 2024143.bz2 ../recovery  
cd ../recovery  
bunzip2 2024144.bz2  
cpio -ivm &lt; 2024144  
/export/home/pserv/ech/ascii\_resend

created a script on /cms/data/ to automatically upload files

check echlog

cat /export/home/pserv/ech/echlog

# Administration, upgrades and maintenances

# Avaya CMS - CMSADM backup to NFS

cat /etc/fstab (to verify NFS share name, in this example is /nfsbu)

[![image.png](https://wiki.tinod.net/uploads/images/gallery/2023-04/scaled-1680-/sUJvAdzkK3seXFB5-image.png)](https://wiki.tinod.net/uploads/images/gallery/2023-04/sUJvAdzkK3seXFB5-image.png)

su root  
cmsadm  
option 3  
then option 2 for network backup  
remote destination type /nfsbu  
wait

  
Backup Log  
/cms/maint/backup/back.log

reboot -n

add cronjob to clean up old backups and to automate cmsadm backups

```
#
# Script to purge old backups
13 6 * * * /export/home/c1as/nfs_cleanup >/dev/null 2>&1
#
# Script to automate CMSADM backups on the first and third Monday of the month
5 3 * * 0 [ `date +\%d` -le 7 ] && /export/home/c1as/cmsadm_backup >/dev/null 2>&1
5 3 * * 0 [[ `date +\%d` -ge 15 && `date +\%d` -le 21 ]] && /export/home/c1as/cmsadm_backup >/dev/null 2>&1

```