root@psa7d's password:

cluster.conf
100% 4442 4.3KB/s 00:00
[root@psa7a cluster]# scp cluster.conf root@psa7dh:/etc/cluster
[root@psa7a cluster]# scp cluster.conf root@psa7d:/etc/clusterhhh
[root@psa7a cluster]# scp cluster.conf root@psa7e:/etc/cluster
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7e's password:
cluster.conf
100% 4442 4.3KB/s 00:00
[root@psa7a cluster]# service ccsd start
Starting ccsd:
[ OK ]
[root@psa7a cluster]# service cman start
Starting cman:
[FAILED]
login as: root
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.

In the course of monitoring individuals improperly using this system, or in

the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@10.123.11.198's password:
Access denied
root@10.123.11.198's password:
Last login: Fri Jan 18 17:25:07 2008 from 10.123.11.211
[root@psa7a ~]# cd /etc/cluster
[root@psa7a cluster]# ll
total 24
-rw-r----- 1 root root 4442 Jan 18 17:24 cluster.conf
-rw-r----- 1 root root 4442 Jan 18 16:59 cluster.conf.bak.1
-rw-r----- 1 root root 4248 Jan 18 16:22 cluster.conf.bak.2
-rw-r----- 1 root root 2099 Jan 18 15:48 cluster.conf.bak.3
-rw-r--r-- 1 root root 34 Jul 18 2007 scsi_watchdog.conf
[root@psa7a cluster]# cd var/logs
-bash: cd: var/logs: No such file or directory
[root@psa7a cluster]# cd var/log
-bash: cd: var/log: No such file or directory
[root@psa7a cluster]# cd /var/log
[root@psa7a log]# ll
total 51860
-rw-r----- 1 root root 1690 Jan 9 20:40 acpid
-rw------- 1 root root 18416 Oct 26 08:48 anaconda.log
-rw------- 1 root root 40100 Oct 26 08:48 anaconda.syslog
-rw------- 1 root root 53779 Oct 26 08:48 anaconda.xlog
drwxr-x--- 2 root root 4096 Mar 6 2007 audit
-rw------- 1 root root
2767 Jan 18 17:34 boot.log
-rw------- 1 root root
9162 Jan 13 04:03 boot.log.1
-rw------- 1 root root
105 Jan 6 04:03 boot.log.2
-rw------- 1 root root
105 Dec 30 04:03 boot.log.3
-rw------- 1 root root
105 Dec 23 04:03 boot.log.4
-rw------- 1 root root 72000 Jan 18 17:30 cron
-rw------- 1 root root 82135 Jan 13 04:03 cron.1
-rw------- 1 root root 85141 Jan 6 04:03 cron.2
-rw------- 1 root root 86156 Dec 30 04:03 cron.3
-rw------- 1 root root 86153 Dec 23 04:03 cron.4
drwxr-xr-x 2 lp sys
4096 Jan 13 04:03 cups

-rw-r--r-- 1 root root 40868 Jan 9 20:40 dmesg

drwxr-xr-x 2 root root
4096 Jan 9 20:44 gdm
-rw-r----- 1 root root
301 Oct 26 16:16 hp_qla2x00_install.log
drwx------ 2 root root
4096 Jan 30 2007 httpd
-r-------- 1 root root 29492 Jan 18 17:34 lastlog
drwxr-xr-x 2 root root
4096 Oct 26 08:42 mail
-rw------- 1 root root 11045 Jan 18 08:45 maillog
-rw------- 1 root root 14974 Jan 13 04:03 maillog.1
-rw------- 1 root root 13929 Jan 6 04:03 maillog.2
-rw------- 1 root root 14021 Dec 30 04:03 maillog.3
-rw------- 1 root root 14021 Dec 23 04:03 maillog.4
-rw-r--r-- 1 root root
0 Oct 26 09:02 mcelog
-rw------- 1 root root 51344694 Jan 18 17:34 messages
-rw------- 1 root root 90703 Jan 13 04:03 messages.1
-rw------- 1 root root
151 Jan 6 04:03 messages.2
-rw------- 1 root root
151 Dec 30 04:03 messages.3
-rw------- 1 root root
151 Dec 23 04:03 messages.4
drwx------ 2 root root
4096 Nov 2 2004 ppp
-rw-r--r-- 1 root root 49260 Jan 18 04:03 prelink.log
-rw-r--r-- 1 root root 29917 Jan 18 04:03 rpmpkgs
-rw-r--r-- 1 root root 29075 Jan 12 04:03 rpmpkgs.1
-rw-r--r-- 1 root root 29075 Jan 5 04:03 rpmpkgs.2
-rw-r--r-- 1 root root 29075 Dec 29 04:03 rpmpkgs.3
-rw-r--r-- 1 root root 29075 Dec 22 04:03 rpmpkgs.4
drwxr-xr-x 2 root root
4096 Jan 18 00:00 sa
drwx------ 2 root root
4096 Feb 9 2007 samba
-rw-r--r-- 1 root root 61190 Oct 26 08:46 scrollkeeper.log
-rw------- 1 root root 21185 Jan 18 17:34 secure
-rw------- 1 root root
852 Jan 9 20:46 secure.1
-rw------- 1 root root
0 Dec 30 04:03 secure.2
-rw------- 1 root root
0 Dec 23 04:03 secure.3
-rw------- 1 root root
0 Dec 16 04:03 secure.4
-rw------- 1 root root
0 Jan 13 04:03 spooler
-rw------- 1 root root
0 Jan 6 04:03 spooler.1
-rw------- 1 root root
0 Dec 30 04:03 spooler.2
-rw------- 1 root root
0 Dec 23 04:03 spooler.3
-rw------- 1 root root
0 Dec 16 04:03 spooler.4
drwxr-x--- 2 squid squid 4096 Jan 22 2007 squid
drwxr-xr-x 2 root root
4096 Oct 5 2004 vbox
-rw-rw-r-- 1 root utmp 92160 Jan 18 17:34 wtmp
-rw-rw-r-- 1 root utmp 16512 Dec 4 13:50 wtmp.1
-rw-r--r-- 1 root root 52636 Jan 9 20:44 Xorg.0.log
-rw-r--r-- 1 root root 52711 Dec 4 13:44 Xorg.0.log.old
[root@psa7a log]# ll -ltr
total 51864
drwxr-xr-x 2 root root
4096 Oct 5 2004 vbox

drwx------ 2 root root

4096 Nov 2 2004 ppp
drwxr-x--- 2 squid squid 4096 Jan 22 2007 squid
drwx------ 2 root root
4096 Jan 30 2007 httpd
drwx------ 2 root root
4096 Feb 9 2007 samba
drwxr-x--- 2 root root 4096 Mar 6 2007 audit
drwxr-xr-x 2 root root
4096 Oct 26 08:42 mail
-rw-r--r-- 1 root root 61190 Oct 26 08:46 scrollkeeper.log
-rw------- 1 root root 53779 Oct 26 08:48 anaconda.xlog
-rw------- 1 root root 40100 Oct 26 08:48 anaconda.syslog
-rw------- 1 root root 18416 Oct 26 08:48 anaconda.log
-rw-r--r-- 1 root root
0 Oct 26 09:02 mcelog
-rw-r----- 1 root root
301 Oct 26 16:16 hp_qla2x00_install.log
-rw-r--r-- 1 root root 52711 Dec 4 13:44 Xorg.0.log.old
-rw-rw-r-- 1 root utmp 16512 Dec 4 13:50 wtmp.1
-rw------- 1 root root
0 Dec 16 04:03 spooler.4
-rw------- 1 root root
0 Dec 16 04:03 secure.4
-rw-r--r-- 1 root root 29075 Dec 22 04:03 rpmpkgs.4
-rw------- 1 root root 14021 Dec 23 04:03 maillog.4
-rw------- 1 root root
0 Dec 23 04:03 spooler.3
-rw------- 1 root root
0 Dec 23 04:03 secure.3
-rw------- 1 root root
151 Dec 23 04:03 messages.4
-rw------- 1 root root 86153 Dec 23 04:03 cron.4
-rw------- 1 root root
105 Dec 23 04:03 boot.log.4
-rw-r--r-- 1 root root 29075 Dec 29 04:03 rpmpkgs.3
-rw------- 1 root root 14021 Dec 30 04:03 maillog.3
-rw------- 1 root root 86156 Dec 30 04:03 cron.3
-rw------- 1 root root
0 Dec 30 04:03 spooler.2
-rw------- 1 root root
0 Dec 30 04:03 secure.2
-rw------- 1 root root
151 Dec 30 04:03 messages.3
-rw------- 1 root root
105 Dec 30 04:03 boot.log.3
-rw-r--r-- 1 root root 29075 Jan 5 04:03 rpmpkgs.2
-rw------- 1 root root 13929 Jan 6 04:03 maillog.2
-rw------- 1 root root 85141 Jan 6 04:03 cron.2
-rw------- 1 root root
0 Jan 6 04:03 spooler.1
-rw------- 1 root root
151 Jan 6 04:03 messages.2
-rw------- 1 root root
105 Jan 6 04:03 boot.log.2
-rw-r--r-- 1 root root 40868 Jan 9 20:40 dmesg
-rw-r----- 1 root root 1690 Jan 9 20:40 acpid
drwxr-xr-x 2 root root
4096 Jan 9 20:44 gdm
-rw-r--r-- 1 root root 52636 Jan 9 20:44 Xorg.0.log
-rw------- 1 root root
852 Jan 9 20:46 secure.1
-rw-r--r-- 1 root root 29075 Jan 12 04:03 rpmpkgs.1
-rw------- 1 root root 14974 Jan 13 04:03 maillog.1
drwxr-xr-x 2 lp sys
4096 Jan 13 04:03 cups
-rw------- 1 root root 82135 Jan 13 04:03 cron.1
-rw------- 1 root root
0 Jan 13 04:03 spooler

-rw------- 1 root root 90703 Jan 13 04:03 messages.1

-rw------- 1 root root
9162 Jan 13 04:03 boot.log.1
drwxr-xr-x 2 root root
4096 Jan 18 00:00 sa
-rw-r--r-- 1 root root 49260 Jan 18 04:03 prelink.log
-rw-r--r-- 1 root root 29917 Jan 18 04:03 rpmpkgs
-rw------- 1 root root 11045 Jan 18 08:45 maillog
-rw------- 1 root root 72000 Jan 18 17:30 cron
-rw-rw-r-- 1 root utmp 92160 Jan 18 17:34 wtmp
-rw------- 1 root root 21185 Jan 18 17:34 secure
-r-------- 1 root root 29492 Jan 18 17:34 lastlog
-rw------- 1 root root
2767 Jan 18 17:34 boot.log
-rw------- 1 root root 51348800 Jan 18 17:35 messages
[root@psa7a log]# vi messages
[root@psa7a log]# ps -ef|grep psaadmin
psaadmin 8092 1 0 Jan17 ?
00:00:00 /usr/aethos/psa/current/bin/pmd -c
/usr/aethos/psa/current/config/proctab -u psaadmin
psaadmin 8093 8092 0 Jan17 ?
00:00:00 errord -nop
psaadmin 8101 8092 0 Jan17 ?
00:00:00 Trantidy T01 - sms
psaadmin 8102 8092 0 Jan17 ?
00:00:03 PERCHG T01
psaadmin 8105 8092 0 Jan17 ?
00:00:00 STAMP -c
psaadmin 8106 8092 0 Jan17 ?
00:00:00 smscsrv -s smsd1 -n 256
psaadmin 8107 8092 0 Jan17 ?
00:00:00 smschnd -s 34003@localhost -p
SMPP=1095@localhost
psaadmin 8109 8092 0 Jan17 ?
00:00:00 statd -d psa_sys_config
psaadmin 8142 8092 0 Jan17 ?
00:00:00 CDR_Server
psaadmin 8156 8092 0 Jan17 ?
00:00:00 prov_server
psaadmin 8157 8092 0 Jan17 ?
00:00:23 TINT T88
psaadmin 8161 8156 0 Jan17 ?
00:00:00 aepython -worker 1
psaadmin 8164 8092 0 Jan17 ?
00:01:07 spbpf F04
psaadmin 8165 8092 0 Jan17 ?
00:00:19 spbpf F88
psaadmin 8166 8092 0 Jan17 ?
00:00:25 spbpf F89
psaadmin 8170 8156 0 Jan17 ?
00:00:00 aepython -worker 2
psaadmin 8196 8156 0 Jan17 ?
00:00:00 aepython -worker 3
psaadmin 8257 8156 0 Jan17 ?
00:00:00 aepython -worker 4
psaadmin 8308 8092 0 Jan17 ?
00:00:22 TINT T89
root 10580 10541 0 17:37 pts/4 00:00:00 grep psaadmin
[root@psa7a log]# su = [
[root@psa7a log]# su - psaadmin
[psaadmin@psa7a ~]$ pmi =[psaadmin@psa7a ~]$ pmi -f list -s
malloc: using debugging hooks
Current date/time: Fri Jan 18 17:37:38 2008
HOST PACKAGE PROCESS PID RTP STATE TIME
psa7a STAMP STAMP 8105 0 Normal 18:31:34 STAMP -c
psa7a cdrd cdr
8142 0 Normal 18:31:35 CDR_Server

psa7a errord errord 8093 0 Normal 18:31:33 errord -nop

psa7a filter F04
8164 0 Normal 18:31:38 spbpf F04
psa7a filter F88
8165 0 Normal 18:31:38 spbpf F88
psa7a filter F89
8166 0 Normal 18:31:38 spbpf F89
psa7a periodic P01
8101 0 Normal 18:31:33 Trantidy T01 - sms
psa7a periodic P02
8102 0 Normal 18:31:33 PERCHG T01
psa7a prov a1
8156 0 Normal 18:31:38 prov_server
psa7a psa T88
8157 0 Normal 18:31:38 TINT T88
psa7a psa T89
8308 0 Normal 18:31:50 TINT T89
psa7a smsd smsd1 8106 0 Normal 18:31:34 smscsrv -s smsd1 -n 256
psa7a smsd smsdh1 8107 0 Normal 18:31:34 smschnd -s 34003@localhost -p
SMPP=1095@localhost
psa7a statd statd 8109 0 Normal 18:31:34 statd -d psa_sys_config
[psaadmin@psa7a ~]$ pmd
malloc: using debugging hooks
[psaadmin@psa7a ~]$ pmdstop
malloc: using debugging hooks
malloc: using debugging hooks
This action shuts the pmd and all application processes.
Are you sure you wish to proceed (y/n) ? y
[psaadmin@psa7a ~]$ pmi -f list -s
malloc: using debugging hooks
Failed to initialise IPC: status -1004, errno 111
[psaadmin@psa7a ~]$ ps -ef | grep psaadmin
malloc: using debugging hooks
malloc: using debugging hooks
root 10581 10541 0 17:37 pts/4 00:00:00 su - psaadmin
psaadmin 10582 10581 0 17:37 pts/4 00:00:00 -bash
psaadmin 10619 10582 0 17:38 pts/4 00:00:00 ps -ef
psaadmin 10620 10582 0 17:38 pts/4 00:00:00 grep psaadmin
[psaadmin@psa7a ~]$ exit
logout
malloc: using debugging hooks
logout
[root@psa7a log]# su - informix
-bash-3.00$ onstat IBM Informix Dynamic Server Version 10.00.FC6
581444 Kbytes

-- On-Line -- Up 1 days 01:57:26 --

-bash-3.00$ onmode -ky

-bash-3.00$ onstat shared memory not initialized for INFORMIXSERVER 'aeshm'
-bash-3.00$ exit
logout
[root@psa7a log]# df

Filesystem
1K-blocks
Used Available Use% Mounted on
/dev/mapper/vg00-lvol1
507748 190550 290984 40% /
/dev/cciss/c0d0p1
256666 21687 221727 9% /boot
none
4081948
0 4081948 0% /dev/shm
/dev/mapper/vg00-lvol3
507748 17641 463893 4% /home
/dev/mapper/vg00-lvol4
4128448 647388 3271348 17% /opt
/dev/mapper/vg00-lvol5
507748 24211 457323 6% /tmp
/dev/mapper/vg00-lvol6
4128448 2570688 1348048 66% /usr
/dev/mapper/vg00-lvol7
253871 10709 230055 5% /usr/local
/dev/mapper/vg00-lvol8
9289080 3506360 5310864 40% /var
/dev/mapper/vg00-aethos
4128448 132700 3786036 4% /usr/aethos
/dev/mapper/vg00-debuglogs
2064208 232344 1727008 12% /usr/aethos/debuglogs
/dev/mapper/vg00-suppimp
253871 10288 230476 5% /usr/aethos/suppimp
/dev/mapper/vg00-customer
253871 10288 230476 5% /usr/aethos/customer
/dev/mapper/vgapp1-datafiles
4128448 44032 3874704 2% /usr/aethos/datafiles
/dev/mapper/vgapp1-suppimpapp1
253871 10288 230476 5% /usr/aethos/suppimpapp1
/dev/mapper/vgdblpd-backups
31729
4512 25579 15% /usr/aethos/backups
/dev/mapper/vgdblpd-suppimpdb
253871 10288 230476 5% /usr/aethos/suppimpdb
[root@psa7a log]# umount /usr/aethos/suppimpdb
[root@psa7a log]# umount /usr/aethos/backups
[root@psa7a log]# df
Filesystem
1K-blocks
Used Available Use% Mounted on
/dev/mapper/vg00-lvol1
507748 190549 290985 40% /
/dev/cciss/c0d0p1
256666 21687 221727 9% /boot
none
4081948
0 4081948 0% /dev/shm
/dev/mapper/vg00-lvol3
507748 17641 463893 4% /home
/dev/mapper/vg00-lvol4
4128448 647388 3271348 17% /opt
/dev/mapper/vg00-lvol5

507748 24211 457323 6% /tmp

/dev/mapper/vg00-lvol6
4128448 2570688 1348048 66% /usr
/dev/mapper/vg00-lvol7
253871 10709 230055 5% /usr/local
/dev/mapper/vg00-lvol8
9289080 3506360 5310864 40% /var
/dev/mapper/vg00-aethos
4128448 132700 3786036 4% /usr/aethos
/dev/mapper/vg00-debuglogs
2064208 232344 1727008 12% /usr/aethos/debuglogs
/dev/mapper/vg00-suppimp
253871 10288 230476 5% /usr/aethos/suppimp
/dev/mapper/vg00-customer
253871 10288 230476 5% /usr/aethos/customer
/dev/mapper/vgapp1-datafiles
4128448 44032 3874704 2% /usr/aethos/datafiles
/dev/mapper/vgapp1-suppimpapp1
253871 10288 230476 5% /usr/aethos/suppimpapp1
[root@psa7a log]# df
Filesystem
1K-blocks
Used Available Use% Mounted on
/dev/mapper/vg00-lvol1
507748 190549 290985 40% /
/dev/cciss/c0d0p1
256666 21687 221727 9% /boot
none
4081948
0 4081948 0% /dev/shm
/dev/mapper/vg00-lvol3
507748 17641 463893 4% /home
/dev/mapper/vg00-lvol4
4128448 647388 3271348 17% /opt
/dev/mapper/vg00-lvol5
507748 24211 457323 6% /tmp
/dev/mapper/vg00-lvol6
4128448 2570688 1348048 66% /usr
/dev/mapper/vg00-lvol7
253871 10709 230055 5% /usr/local
/dev/mapper/vg00-lvol8
9289080 3506360 5310864 40% /var
/dev/mapper/vg00-aethos
4128448 132700 3786036 4% /usr/aethos
/dev/mapper/vg00-debuglogs
2064208 232344 1727008 12% /usr/aethos/debuglogs
/dev/mapper/vg00-suppimp
253871 10288 230476 5% /usr/aethos/suppimp
/dev/mapper/vg00-customer
253871 10288 230476 5% /usr/aethos/customer
/dev/mapper/vgapp1-datafiles

4128448 44032 3874704 2% /usr/aethos/datafiles

/dev/mapper/vgapp1-suppimpapp1
253871 10288 230476 5% /usr/aethos/suppimpapp1
[root@psa7a log]# service ccsd status
ccsd (pid 10469) is running...
[root@psa7a log]# service cman status
Protocol version: 5.0.1
Config version: 6
Cluster name: alpha_cluster
Cluster ID: 50356
Cluster Member: Yes
Membership state: Cluster-Member
Nodes: 1
Expected_votes: 5
Total_votes: 1
Quorum: 3 Activity blocked
Active subsystems: 0
Node name: psa7a
Node ID: 1
Node addresses: 10.123.1.198
[root@psa7a log]# service cman start
Starting cman:
[root@psa7a log]# service fenced start
Starting fence domain:

[ OK ]

[root@psa7a ~]# ssh psa7b

************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.

************************************************************************
****
root@psa7b's password:
Last login: Mon Jan 21 08:46:13 2008 from 10.122.17.177
[root@psa7b ~]# service ccsd start
Starting ccsd:
[ OK ]
[root@psa7b ~]# service cman start
Starting cman:
[ OK ]
[root@psa7b ~]# exit
logout
Connection to psa7b closed.
[root@psa7a ~]# ssh psa7c
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7c's password:
Last login: Mon Jan 21 00:33:10 2008 from 10.122.17.177
[root@psa7c ~]# service ccsd start
Starting ccsd:
[ OK ]
[root@psa7c ~]# service cman start
Starting cman:
[ OK ]
[root@psa7c ~]# exit
logout
Connection to psa7c closed.
[root@psa7a ~]# ssh psa7d
************************************************************************
***
NOTICE TO USERS

This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7d's password:
Last login: Mon Jan 21 08:48:40 2008 from 10.122.17.177
[root@psa7d ~]# service ccsd start
Starting ccsd:
[ OK ]
[root@psa7d ~]# service cman start
Starting cman:
[ OK ]
[root@psa7d ~]# exit
logout
Connection to psa7d closed.
[root@psa7a ~]# ssh psa7e
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****

root@psa7e's password:
Last login: Mon Jan 21 08:50:00 2008 from 10.122.17.177
[root@psa7e ~]# service ccsd start
Starting ccsd:
[ OK ]
[root@psa7e ~]# service cman start
Starting cman:
[ OK ]
[root@psa7e ~]# exit
logout
Connection to psa7e closed.
[root@psa7a ~]# service fenced start
Starting fence domain:
[ OK ]
[root@psa7a ~]# service rgmanager start
Starting Cluster Service Manager:
[root@psa7a ~]# ssh psa7b
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7b's password:
Last login: Mon Jan 21 08:58:22 2008 from psa7a
[root@psa7b ~]# service fenced start
Starting fence domain:
[ OK ]
[root@psa7b ~]# exit
logout
Connection to psa7b closed.
[root@psa7a ~]# ssh psa7c
************************************************************************
***
NOTICE TO USERS

This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7c's password:
Last login: Mon Jan 21 00:44:32 2008 from psa7a
[root@psa7c ~]# service fenced start
Starting fence domain:
[ OK ]
[root@psa7c ~]# exit
logout
Connection to psa7c closed.
[root@psa7a ~]# ssh psa7d
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7d's password:
Last login: Mon Jan 21 08:59:18 2008 from psa7a

[root@psa7d ~]# service fenced start

Starting fence domain:
[ OK ]
[root@psa7d ~]# exit
logout
Connection to psa7d closed.
[root@psa7a ~]# ssh psa7e
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7e's password:
Last login: Mon Jan 21 08:59:41 2008 from psa7a
[root@psa7e ~]# service fenced start
Starting fence domain:
[ OK ]
[root@psa7e ~]# exit
logout
Connection to psa7e closed.
[root@psa7a ~]# service rgmanager start
Starting Cluster Service Manager:
[root@psa7a ~]# service clvmd start
Activating VGs:
[ OK ]
[root@psa7a ~]# ssh psa7b
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.

In the course of monitoring individuals improperly using this system, or in

the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7b's password:
Last login: Mon Jan 21 09:00:56 2008 from psa7a
[root@psa7b ~]# service clvmd start
Activating VGs:
[ OK ]
[root@psa7b ~]# exit
logout
Connection to psa7b closed.
[root@psa7a ~]# ssh psa7b
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7b's password:
Last login: Mon Jan 21 09:04:06 2008 from psa7a
[root@psa7b ~]# exit
logout
Connection to psa7b closed.
[root@psa7a ~]# ssh psa7c

************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7c's password:
Last login: Mon Jan 21 00:46:46 2008 from psa7a
[root@psa7c ~]# service clvmd start
Activating VGs:
[ OK ]
[root@psa7c ~]# exit
logout
Connection to psa7c closed.
[root@psa7a ~]# ssh psa7d
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.

************************************************************************
****
root@psa7d's password:
Last login: Mon Jan 21 09:01:25 2008 from psa7a
[root@psa7d ~]# service clvmd start
Activating VGs:
[ OK ]
[root@psa7d ~]# exit
logout
Connection to psa7d closed.
[root@psa7a ~]# ssh psa7e
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7e's password:
Last login: Mon Jan 21 09:01:40 2008 from psa7a
[root@psa7e ~]# service clvmd start
Activating VGs:
[ OK ]
[root@psa7e ~]# exit
logout
Connection to psa7e closed.
[root@psa7a ~]# service rgmanager start
Starting Cluster Service Manager:
[root@psa7a ~]# cat /var/log
cat: /var/log: Is a directory
[root@psa7a ~]# cd /var
[root@psa7a var]# ls
account crash depots gdm local log
mail opt
run tmp www
cache db empty lib lock lost+found nis preserve spool tux yp
[root@psa7a var]# cd logs

-bash: cd: logs: No such file or directory

[root@psa7a var]# cd log
[root@psa7a log]# ls
acpid
cron
lastlog messages.2 sa
spooler.2
anaconda.log cron.1
mail
messages.3 samba
spooler.3
anaconda.syslog cron.2
maillog messages.4 scrollkeeper.log spooler.4
anaconda.xlog cron.3
maillog.1 ppp
secure
squid
audit
cron.4
maillog.2 prelink.log secure.1
vbox
boot.log
cups
maillog.3 rpmpkgs
secure.2
wtmp
boot.log.1
dmesg
maillog.4 rpmpkgs.1 secure.3
wtmp.1
boot.log.2
gdm
mcelog
rpmpkgs.2 secure.4
Xorg.0.log
boot.log.3
hp_qla2x00_install.log messages rpmpkgs.3 spooler
Xorg.0.log.old
boot.log.4
httpd
messages.1 rpmpkgs.4 spooler.1
[root@psa7a log]# tail messages
Jan 21 14:55:19 psa7a clurgmgrd[31429]: <notice> Service psa7_db is stopped
Jan 21 15:00:28 psa7a fenced[1513]: fenced is already running
Jan 21 15:00:28 psa7a fenced: startup succeeded
Jan 21 15:03:33 psa7a kernel: cdrom: open failed.
Jan 21 15:03:39 psa7a vgchange: 4 logical volume(s) in volume group "vgdbhpd" now
active
Jan 21 15:03:39 psa7a vgchange: 0 logical volume(s) in volume group "vgdbll" now
active
Jan 21 15:03:39 psa7a vgchange: 2 logical volume(s) in volume group "vgapp1" now
active
Jan 21 15:03:39 psa7a vgchange: 12 logical volume(s) in volume group "vg00" now
active
Jan 21 15:03:39 psa7a vgchange: 15 logical volume(s) in volume group "vgdblpd" now
active
Jan 21 15:03:39 psa7a clvmd: Activating VGs: succeeded
[root@psa7a log]# ssh psa7c
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal

activity, system personnel may provide the evidence of such monitoring to

law enforcement officials.
************************************************************************
****
root@psa7c's password:
Last login: Mon Jan 21 00:50:36 2008 from psa7a
[root@psa7c ~]# service rgmanager start
Starting Cluster Service Manager:
[root@psa7c ~]# tail /var/log/messages
Jan 21 00:46:54 psa7c fenced[5315]: fenced is already running
Jan 21 00:46:54 psa7c fenced: startup succeeded
Jan 21 00:50:36 psa7c sshd(pam_unix)[5346]: session opened for user root by
root(uid=0)
Jan 21 00:50:38 psa7c kernel: cdrom: open failed.
Jan 21 00:50:43 psa7c vgchange: 0 logical volume(s) in volume group "vgdbll" now
active
Jan 21 00:50:43 psa7c vgchange: 2 logical volume(s) in volume group "vgapp1" now
active
Jan 21 00:50:43 psa7c vgchange: 12 logical volume(s) in volume group "vg00" now
active
Jan 21 00:50:43 psa7c vgchange: 15 logical volume(s) in volume group "vgdblpd" now
active
Jan 21 00:50:43 psa7c clvmd: Activating VGs: succeeded
Jan 21 00:56:15 psa7c sshd(pam_unix)[5430]: session opened for user root by
root(uid=0)
[root@psa7c ~]# ps -ef|grep rgmanager
root
5476 5430 0 00:56 pts/1 00:00:00 grep rgmanager
[root@psa7c ~]# exit
logout
Connection to psa7c closed.
[root@psa7a log]# ssh psa7c
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is

advised that if such monitoring reveals possible evidence of criminal

activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7c's password:
Last login: Mon Jan 21 00:56:15 2008 from psa7a
[root@psa7c ~]# service rgmanager start
Starting Cluster Service Manager:
[root@psa7c ~]# exit
logout
Connection to psa7c closed.
[root@psa7a log]# ssh psa7e
************************************************************************
***
NOTICE TO USERS
This system is for the use of authorised users only. Individuals using
this computer system without authority, on in excess of their authority,
are subject to having all of their activities on this system monitored and
recorded by system personnel.
In the course of monitoring individuals improperly using this system, or in
the course of system maintenance, the activities of authorized users may
also be monitored.
Anyone using this system expressly consents to such monitoring and is
advised that if such monitoring reveals possible evidence of criminal
activity, system personnel may provide the evidence of such monitoring to
law enforcement officials.
************************************************************************
****
root@psa7e's password:
Last login: Mon Jan 21 09:07:32 2008 from psa7a
[root@psa7e ~]# service rgmanager start
Starting Cluster Service Manager:
[root@psa7e ~]# clustat
Member Status: Quorate
Member Name
------ ---aepsa7a
aepsa7b
aepsa7c

Status
-----Online, rgmanager
Online, rgmanager
Online, rgmanager

aepsa7d
aepsa7e

Online, rgmanager
Online, Local, rgmanager

Service Name
Owner (Last)
------- -------- -----psa7_db
(none)
psa7_app1
(none)
psa7_app2
(none)
psa7_app3
(none)
[root@psa7e ~]#

State
----stopped
stopped
stopped
stopped

[root@psa7a dev]# lvcreate -L 1536 -n cdr /dev/vgapp1

Logical volume "cdr" created
[root@psa7a dev]# gfs_mkfs
gfs_mkfs: no device specified (try -h for help)
[root@psa7a dev]# gfs_mkfs -h
Usage:
gfs_mkfs [options] <device>
Options:
-b <bytes>
Filesystem block size
-D
Enable debugging code
-h
Print this help, then exit
-J <MB>
Size of journals
-j <num>
Number of journals
-O
Don't ask for confirmation
-p <name>
Name of the locking protocol
-q
Don't print anything
-r <MB>
Resource Group Size
-s <blocks>
Journal segment size
-t <name>
Name of the lock table
-V
Print program version information, then exit
[root@psa7a dev]# ll /dev/vgapp1
total 0
lrwxrwxrwx 1 root root 22 Jan 21 15:45 cdr -> /dev/mapper/vgapp1-cdr
lrwxrwxrwx 1 root root 28 Jan 9 20:40 datafiles -> /dev/mapper/vgapp1-datafiles
lrwxrwxrwx 1 root root 30 Jan 9 20:40 suppimpapp1 -> /dev/mapper/vgapp1suppimpapp1
[root@psa7a dev]# gfs_mkfs -p lock_dlm -t psa7:cdr -j 5 (5 for 5
nodes)/dev/mapper/vgapp1-cdr
This will destroy any data on /dev/mapper/vgapp1-cdr.
Are you sure you want to proceed? [y/n] y

Device:
/dev/mapper/vgapp1-cdr
Blocksize:
4096
Filesystem Size:
262096
Journals:
4
Resource Groups:
8
Locking Protocol:
lock_dlm
Lock Table:
psa7:cdr
Syncing...
All Done
[root@psa7a dev]# lvextend -L +512 /dev/vgapp1/suppimpapp1
Extending logical volume suppimpapp1 to 768.00 MB
Logical volume suppimpapp1 successfully resized
[root@psa7a dev]# extendfs
-bash: extendfs: command not found
[root@psa7a dev]# clear
[root@psa7a dev]# gfs_mkfs -p lock_dlm -t psa7:suppimpapp1 -j 5 /dev/mapper/vgapp1suppimpapp1
This will destroy any data on /dev/mapper/vgapp1-suppimpapp1.
It appears to contain a EXT2/3 filesystem.
Are you sure you want to proceed? [y/n] y
Device:
/dev/mapper/vgapp1-suppimpapp1
Blocksize:
4096
Filesystem Size:
65492
Journals:
4
Resource Groups:
8
Locking Protocol:
lock_dlm
Lock Table:
psa7:suppimpapp1
Syncing...
All Done
If face the following issue:
[root@psa7a dev]# gfs_mkfs -p lock_dlm -t psa7:suppimpapp1 -j 5 /dev/mapper/vgapp1suppimpapp1
gfs_mkfs: Partition too small for number/size of journals
--- Logical volume --LV Name
/dev/vgapp1/suppimpapp1
VG Name
vgapp1

LV UUID
Qt2sxE-rg1O-8P3T-1nuh-mVKl-7k88-6fnhN5
LV Write Access
read/write
LV Status
available
# open
0
LV Size
256.00 MB
Current LE
64
Segments
1
Allocation
inherit
Read ahead sectors 0
Block device
253:24
[root@psa7a dev]# lvextend -L +512 /dev/vgapp1/suppimpapp1
Extending logical volume suppimpapp1 to 768.00 MB
Logical volume suppimpapp1 successfully resized
[root@psa7a dev]# gfs_mkfs -p lock_dlm -t psa7:suppimpapp1 -j 4
/dev/mapper/vgapp1-suppimpapp1
This will destroy any data on /dev/mapper/vgapp1-suppimpapp1.
It appears to contain a EXT2/3 filesystem.
Are you sure you want to proceed? [y/n] y
Device:
/dev/mapper/vgapp1-suppimpapp1
Blocksize:
4096
Filesystem Size:
65492
Journals:
4
Resource Groups:
8
Locking Protocol:
lock_dlm
Lock Table:
psa7:suppimpapp1
Syncing...
All Done
[root@psa7a dev]# gfs_mkfs -p lock_dlm -t psa7:datafiles -j 5 /dev/mapper/vgapp1datafiles
This will destroy any data on /dev/mapper/vgapp1-datafiles.
It appears to contain a EXT2/3 filesystem.
Are you sure you want to proceed? [y/n] y
Device:
/dev/mapper/vgapp1-datafiles
Blocksize:
4096
Filesystem Size:
917396
Journals:
4
Resource Groups:
14
Locking Protocol:
lock_dlm

Lock Table:

psa7:datafiles

Syncing...
All Done
Do the below steps on all nodes:
[root@psa7b ~]# vgchange -a n vgapp1
0 logical volume(s) in volume group "vgapp1" now active
[root@psa7b ~]# vgchange -a y vgapp1
3 logical volume(s) in volume group "vgapp1" now active
[root@psa7a ~]# mount -t gfs /dev/vgapp1/suppimpapp1 /usr/aethos/suppimpapp1
[root@psa7a ~]# mount -t gfs /dev/vgapp1/datafiles /usr/aethos/datafiles
[root@psa7a ~]# ll /usr/aethos/cdr
total 0
[root@psa7a ~]# mount -t gfs /dev/vgapp1/cdr /usr/aethos/cdr
[root@psa7a ~]# df
Filesystem
1K-blocks
Used Available Use% Mounted on
/dev/mapper/vg00-lvol1
507748 190579 290955 40% /
/dev/cciss/c0d0p1
256666 21687 221727 9% /boot
none
4081948
0 4081948 0% /dev/shm
/dev/mapper/vg00-lvol3
507748
/dev/mapper/vg00-lvol4
4128448
/dev/mapper/vg00-lvol5
507748
/dev/mapper/vg00-lvol6
4128448
/dev/mapper/vg00-lvol7
253871

17641

463893 4% /home

647388 3271348 17% /opt

24245

457289 6% /tmp

2570688 1348048 66% /usr

10709

230055 5% /usr/local

/dev/mapper/vg00-lvol8
9289080 3508628 5308596 40% /var
/dev/mapper/vg00-aethos
4128448 135040 3783696 4% /usr/aethos
/dev/mapper/vg00-debuglogs
2064208 35920 1923432 2% /usr/aethos/debuglogs
/dev/mapper/vg00-suppimp
253871 10288 230476 5% /usr/aethos/suppimp
/dev/mapper/vg00-customer
253871 10288 230476 5% /usr/aethos/customer

/dev/mapper/vgapp1-suppimpapp1
261968
20 261948 1% /usr/aethos/suppimpapp1
/dev/mapper/vgapp1-datafiles
3669584
20 3669564 1% /usr/aethos/datafiles
/dev/mapper/vgapp1-cdr
1048384
20 1048364 1% /usr/aethos/cdr