Sunday, March 25, 2012

Oracle (10gR2) RAC on Itanium 64 bit System and RHEL 4 Update 4


Operating System        :  RHEL 4 U 6
Architecture                :  ia64 (Itanium 64 bit System)
RAM                          :  8GB
/oracle                         :  Oracle BASE
/database                     :  Oracle database – OCFS2   (/dev/mapper/disk1p1)
/archives                      :  Archives – OCFS2               (/dev/mapper/disk2p1)
db_name                     : hms
single instance              : hms
multiple instance          : hms1, hms2

First set up Oracle DB in single instance mode and then to Multiple instance.

1). /etc/hosts                                        (Node 1 & 2)

127.0.0.1                     localhost.localdomain             localhost
172.17.2.165               rh-rac-srv1.hq.mh                   rh-rac-srv1
172.17.2.166               rh-rac-srv2.hq.mh                   rh-rac-srv2
172.17.2.167               rh-rac-vip1.hq.mh                   rh-rac-vip1
172.17.2.168                          rh-rac-vip2.hq.mh                   rh-rac-vip2
10.10.10.1                   rh-rac-prv1.hq.mh                   rh-rac-prv1
10.10.10.2                   rh-rac-prv2.hq.mh                   rh-rac-prv2

2). /etc/sysconfig/network                   (Node 1)

NETWORKING=yes
HOSTNAME=rh-rac-srv1

3). /etc/sysconfig/network                   (Node 2)

NETWORKING=yes
HOSTNAME=rh-rac-srv2

# hostname                                          (Check)
rh-rac-srv1

4).  Create oracle users                        (Node 1 & 2)

#  groupadd –g 400 dba
#  useradd –g 400 –u 400 oracle
#  passwd oracle

5). Set environment variables in .bash_profile            (Node 1 & 2)

The commented environment variables should be set only after oracle installation, before configuring the database…

ORACLE_BASE=/oracle
#CRS_HOME=$ORACLE_BASE/product/10.2.0/crs
#RDBMS_HOME=$ORACLE_BASE/product/10.2.0/db_1
#ORACLE_HOME=$RDBMS_HOME
#PATH=$PATH:$HOME/bin:$RDBMS_HOME/bin:$CRS_HOME/bin
#ORACLE_SID=hms1 ( hms2 in Node 2)
#TNS_ADMIN=$RDBMS_HOME/network/admin
#NLS_LANG='american_america.we8iso8859p1'

#LD_LIBRARY_PATH=$ORACLE_HOME/lib

export ORACLE_BASE
#export RDBMS_HOME
#export ORACLE_HOME
#export CRS_HOME
#export LD_LIBRARY_PATH
#export ORACLE_SID
export PATH
#export TNS_ADMIN
#export NLS_LANG
unset USERNAME
umask 022

6). # chown oracle.dba /oracle –R                  (Node 1 & 2)

7). Install rpms                                                (Node 1 & 2)

a.       ocfs2
b.      ocfs2tools
c.       osfs2console
d.      ocfs2devel

8). Configure ssh (Node 1 & 2) – The important thing is the same node should be connected without password after configuring ssh for both root and oracle.. Example from rh-rac-srv1

$ ssh rh-rac-srv1

This should work without password…

9). #  ocfs2console
           
           » Cluster » Configure Nodes

Note : If you get the error (bug) : ‘o2cb_ctl : Unable to access cluster service while creating node’, do the following
           
# /etc/init.d/o2cb offline ocfs2
# /etc/init.d/o2cb unload
# rm /etc/ocfs2/cluster.conf
# /rm –rf /config

Repeat step 9 in node 2

# ocfs2console » Cluster » Configure Nodes » Add

            rh-rac-srv1       172.17.2.165   » Apply
            rh-rac-srv2       172.17.2.166   » Apply

Check /etc/ocfs2/cluster.conf

10). # /etc/init.d/o2cb  enable

11). # mkdir /database
       # mkdir /archives

12). # mkfs.ocfs2        -b 4K   -C 32K -N 4 –L /database /dev/mapper/disk1p1
       # mkfs.ocfs2        -b 4K   -C 32K -N 4 –L /archives /dev/mapper/disk2p1

Note : If you get the error : ocfs2-inititalize-super : 1455 Error : Couldn't mount because of   unsupported optional feature (50), use --fs-feature-level=max-compat option...

# mkfs.ocfs2 -b 4K   -C 32K -N 4 –L /database /dev/mapper/disk1p1
                                                                                        --fs-feature-level=max-compat
# mkfs.ocfs2  -b 4K   -C 32K -N 4 –L /archives /dev/mapper/disk2p1
                                                                                        --fs-feature-level=max-compat

13). # mount –t ocfs2 /dev/mapper/disk1p1   /database
       # mount –t ocfs2 /dev/mapper/disk2p1   /archives

If everything is OK

14). # umount /database
       # umount /archives

15). # mount –t ocfs2 –o datavolume,nointr /dev/mapper/disk1p1 /database
       # mount –t ocfs2 –o datavolume,nointr /dev/mapper/disk2p1 /archieves

16). Add the below 2 lines in /etc/fstab          (Node 1 & 2)

/dev/mapper/disk1p1   /database         ocfs2   -netdev,datavolume,nointr      0 0
/dev/mapper/disk2p1   /archives          ocfs2   -netdev,datavolume,nointr      0 0

17). # chkconfig --add osfs2
       # chkconfig --add o2cb

18). # /etc/init.d/o2cb configure » y » » ocfs2 » » 

19). # mkdir /database/oracrs
       # mkkdir /database/hms

20). Add to /etc/sysctl.conf

net.core.rmem_default = 534288
net.core.rmem_max = 534288
net.core.wmem_default = 534288
net.core.wmem_max = 534288
kernel.sysrq = 0
kernel.shmmax = 5368709120            (Out of 8 GB RAM, 5 GB is allocated)
kernel.shmmni = 4096
kernel.shmall = 2097152
kernel.sem = 300 45000 300 150
fs.file-max = 65536

21).  Add to /etc/limits.conf

            oracle           hard        nproc           16384
oracle           soft         nproc           16384
oracle           hard        nofile          65536
oracle           soft         nofile          65536
oracle           soft         memlock    3145728
oracle           hard        memlock    3145728
 
22). Install Oracle Clusterware 10.2.0.1 in a separate home. (/oracle/product/10.2.0/crs)

If you get : ‘An unexpected error has been detected by HotSpot Virtual Machine’ error (bug) , install Oracle patch 5390722 (JDK & JRE) in separate home, first. And then install Oracle Clusterware with –jreLoc option. (Reference : Doc ID – 391844.1, 401332.1, 400227.1)

Eg : Assume that you have installed the 5390722 patch in /oracle/product/jre/jre/ folder :

/database//media/cdrom/runinstaller –jreLoc /oracle/product/jre/jre/1.4.2


23). Install libaio-0.3.105-2.ia64.rpm & libaio-devel (3rd Linux CD)


24). Install oracle in separate home (/oracle/product/10.2.0/db_1) using –jreLoc option.

Eg : $/media/cdrom/runinstaller –jreLoc /oracle/product/jre/jre/1.4.2


25). Change  JREDIR variable in /oracle/product/10.2.0/crs/bin/vipca to  /oracle/product/jre/jre/1.4.2 and install vipca.


26). Change JREDIR vairable in /oracle/product/10.2.0/crs/bin/cluvfy to
/oracle/product/jre/jre/1.4.2 and execute it.

Eg : $./cluvfy stage –post crsinst –n rh-rac-srv1, rh-rac-srv2


27). Install Oracle Patch (5337014) 10.2.0.3 using the –jreLoc option.

Eg : ./runinstaller –jreLoc /oracle/product/10.2.0/jre/jre/1.4.2

First patch the crs home and then database home


28).  Set & starting up Oracle Single Instance

a.       Copy the Hot Backup & archives
b.      Edit init.ora file & create spfile
c.       Create required folders such as adump, bdump, etc
d.      Edit .bash_profile (add TNS_NAMES & ORACLE_HOME etc)
e.       Configure listener.ora & tnsnames.ora

Recover the database :

Startup mount;
Alter database recover database using backup controlfile until cancel;
Shutdown immediate;
Startup upgrade;
@/oracle/product/10.2.0/db_1/rdbms/admin/utlirp.sql
Shutdown immediate;
Startup;
@/oracle/product/10.2.0/db_1/rdbms/admin/utlrp.sql
Alter tablespace undotbs rename to undotbs1;
Alter system set undo_tablespace=undotbs1;
Shutdown immediate;


28).  Set & starting up Oracle Multiple Instance

a.       Edit listener.ora & tnsnames.ora  (execute netca using –jreLoc option)
b.      ln –s spfilehms.ora spfilehms1.ora            (Node 1)
c.       ln –s hms hms1                                         (Node 1)
d.      ln –s spfilehms.ora spfilehms2.ora            (Node 2)
e.       ln –s hms hms2                                          (Node 2)
f.       Configure init.ora & create new spfile in both nodes.
g.      Sqlplus / as sysdba
h.      Startup;
i.        Spool catclust.log
j.        @catclust.sql
k.      spool off
l.        shutdown immediate;
m.    # srvctl add database –d hms –o $ORCLE_HOME
n.      # srvctl add instance –d hms –i hms1 –n rh-rac-srv1
o.      # srvctl add instance –d hms –i hms2 –n rh-rac-srv2
p.      # srvctl start instance –d hms –i hms1
q.      sqlplus / as sysdba
r.        startup;
s.       create redo logs for thread 2
t.        alter database enable thread 2;
u.      create undo tablespace undotbs2
v.      # srvctl start instance –h hms –i hms2
w.    connect to db using ‘hms’ as connect string & check the load balance;

 

Sample listener.ora

# listener.ora.rh-rac-srv1 Network Configuration File: /oracle/product/10.2.0/db_1/network/admin/listener.ora.rh-rac-srv1
# Generated by Oracle configuration tools.

SID_LIST_LISTENER_RH-RAC-SRV1 =
  (SID_LIST =
    (SID_DESC =
      (SID_NAME = PLSExtProc)
      (ORACLE_HOME = /oracle/product/10.2.0/db_1)
      (PROGRAM = extproc)
    )
  )

LISTENER_RH-RAC-SRV1 =
  (DESCRIPTION_LIST =
    (DESCRIPTION =
      (ADDRESS = (PROTOCOL = TCP)(HOST = rh-rac-vip1)(PORT = 1521)(IP = FIRST))
      (ADDRESS = (PROTOCOL = TCP)(HOST = 172.17.2.165)(PORT = 1521)(IP = FIRST))
    )
  )

 

Sample tnsname.ora

# tnsnames.ora.mh-rac-srv1 Network Configuration File: /oracle/oracle/product/10.2.0/db_1/network/admin/tnsnames.ora.mh-rac-srv1
# Generated by Oracle configuration tools.

LISTENER_HMS =
    (ADDRESS_LIST =
      (ADDRESS = (PROTOCOL = TCP)(HOST = 172.17.2.167)(PORT = 1521))
    )
LISTENER_HMS_REMOTE =
    (ADDRESS_LIST =
      (ADDRESS = (PROTOCOL = TCP)(HOST = 172.17.2.167)(PORT = 1521))
      (ADDRESS = (PROTOCOL = TCP)(HOST = 172.17.2.168)(PORT = 1521))
    )

HMS =
  (DESCRIPTION =
    (ADDRESS_LIST =
      (ADDRESS = (PROTOCOL = TCP)(HOST = rh-rac-vip1)(PORT = 1521))
      (ADDRESS = (PROTOCOL = TCP)(HOST = rh-rac-vip2)(PORT = 1521))
    )
    (LOAD_BALANCE=yes)
    (CONNECT_DATA =
      (SERVICE_NAME = hms.hq.mh)
      (FAILOVER_MODE=
           (TYPE=SELECT)
           (METHOD=BASIC)
           (DELAY=5)
           (RETRIES=200)
      )
    )
  )

HMS1 =
  (DESCRIPTION =
    (ADDRESS_LIST =
      (ADDRESS = (PROTOCOL = TCP)(HOST = rh-rac-vip1)(PORT = 1521))
    )
    (CONNECT_DATA =
      (SERVICE_NAME = hms.hq.mh)
      (INSTANCE_NAME = hms1)
    )
  )
HMS2 =
  (DESCRIPTION =
    (ADDRESS_LIST =
      (ADDRESS = (PROTOCOL = TCP)(HOST = rh-rac-vip2)(PORT = 1521))
    )
    (CONNECT_DATA =
      (SERVICE_NAME = hms.hq.mh)
      (INSTANCE_NAME = hms2)
    )
  )

EXTPROC_CONNECTION_DATA =
  (DESCRIPTION =
    (ADDRESS_LIST =
      (ADDRESS = (PROTOCOL = IPC)(KEY = EXTPROC0))
    )
    (CONNECT_DATA =
      (SID = PLSExtProc)
      (PRESENTATION = RO)
    )
  )

 

Sample init.ora

*.cluster_database_instances=2
*.cluster_database=true
*.remote_listener='LISTENER_HMS'
hms1.thread=1
hms2.thread=2
hms1.instance_number=1
hms2.instance_number=2
hms1.instance_name=hms1
hms2.instance_name=hms2
*._spin_count=512
*.audit_file_dest='/oracle/admin/hms/adump'
*.audit_trail='db_extended'
*.background_dump_dest='/oracle/admin/hms/bdump'
*.compatible='10.2.0.3.0'
*.control_file_record_keep_time=30
*.control_files='/database/hms/control01.ctl','/database/hms/control02.ctl','/database/hms/control03.ctl'
*.core_dump_dest='/oracle/admin/hms/cdump'
*.cursor_sharing='exact'
*.db_block_checking='TRUE'
*.db_block_checksum='TRUE'
*.db_block_size=8192
*.db_cache_advice='on'
*.db_domain='hq.mh'
*.db_file_multiblock_read_count=8
*.db_files=75
*.db_flashback_retention_target=10080
*.db_name='hms'
*.db_recovery_file_dest='/database/hms/flashdb'
*.db_recovery_file_dest_size=10737418240
*.db_unique_name='hms'
*.fast_start_mttr_target=45
*.fast_start_parallel_rollback='low'
*.global_names=FALSE
*.job_queue_processes=2
*.local_listener='listener_hms'
*.log_archive_dest_1='LOCATION=/archives/hms/archive1
                    VALID_FOR=(ALL_LOGFILES,ALL_ROLES)
                    DB_UNIQUE_NAME=hms
                    MANDATORY
                    REOPEN=60'
*.log_archive_dest_2='LOCATION=/archives/hms/archive2
                    VALID_FOR=(ONLINE_LOGFILES,PRIMARY_ROLE)
                    DB_UNIQUE_NAME=hms
                    MANDATORY
                    REOPEN=60'
*.log_archive_dest_state_1='enable'
*.log_archive_dest_state_2='enable'
*.log_archive_format='hms%s_%t_%r.arc'
*.log_archive_max_processes=10
*.log_archive_min_succeed_dest=2
*.log_buffer=10485760
*.max_dump_file_size='3m'
*.nls_date_format='DD/MM/RRRR'
*.nls_territory='AMERICA'
*.open_cursors=600
*.optimizer_features_enable='10.2.0.1'
*.optimizer_index_caching=25
*.optimizer_index_cost_adj=75
*.optimizer_mode='first_rows_100'
*.parallel_max_servers=16
*.pga_aggregate_target=1048576000
*.processes=75
*.remote_login_passwordfile='EXCLUSIVE'
*.local_listener='listener_hms_remote'
*.resource_limit=true
*.service_names='hms.hq.mh'
*.session_cached_cursors=200
*.sga_max_size=5368709120
*.sga_target=5368709120
*.star_transformation_enabled='true'
*.statistics_level='typical'
*.undo_management='AUTO'
*.undo_retention=0
hms1.undo_tablespace='UNDOTBS1'
hms2.undo_tablespace='UNDOTBS2'
*.user_dump_dest='/oracle/admin/hms/udump'

***
Tested long back,  someday on August, 2008. Just sharing the information !