About DATABASE

19C(19.3) RAC 노 패치 기준 테스트 

참조 : How to Reinstall Oracle Grid Infrastructure Without Disturbing RDBMS Installation (문서 ID 1276975.1)

0. RAC 현재 상태

-- 패치 정보

[racdb1:/home/oracle]> opatch lspatches
29585399;OCW RELEASE UPDATE 19.3.0.0.0 (29585399)
29517242;Database Release Update : 19.3.0.0.190416 (29517242)

-- voting disk 정보

[racdb1:/home/oracle]> crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   a2cd2b1080754f3dbfd1dd9cfc288076 (/dev/oracleasm/disks/CRS1) [CRS]
 2. ONLINE   3d09921f15d14f46bf07b9802ba755d4 (/dev/oracleasm/disks/CRS3) [CRS]
 3. ONLINE   4b80c0d6281c4f13bfbb26f44031a142 (/dev/oracleasm/disks/CRS2) [CRS]
3개의 선호 디스크를 찾았습니다.
[root@rac1 disks]# olsnodes -n -i -s -t
rac1    1       <none>  Active  Unpinned
rac2    2       <none>  Active  Unpinned
[racdb1:/home/oracle]> ocrcheck
Status of Oracle Cluster Registry is as follows :
         Version                  :          4
         Total space (kbytes)     :     491684
         Used space (kbytes)      :      84292
         Available space (kbytes) :     407392
         ID                       :  269554171
         Device/File Name         :       +CRS
                                    Device/File integrity check succeeded
                                    Device/File not configured
                                    Device/File not configured
                                    Device/File not configured
                                    Device/File not configured
         Cluster registry integrity check succeeded
         Logical corruption check bypassed due to non-privileged user

-- disk 정보

[root@rac1 ~]# cd /dev/oracleasm/disks

[root@rac1 disks]# ls
ASM01  ASM02  ASM03  ASM04  ASM05  CRS1  CRS2  CRS3  CRS4  DATA  DATA2  FRA

[root@rac1 disks]# crsctl stat res -t
--------------------------------------------------------------------------------
Name           Target  State        Server                   State details       
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
ora.chad
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
ora.net1.network
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
ora.ons
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMDATA.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        ONLINE  OFFLINE                               STABLE
ora.CRS.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.DATA.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.FRA.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       rac2                     STABLE
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       rac1                     STABLE
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       rac1                     STABLE
ora.asm(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     Started,STABLE
      2        ONLINE  ONLINE       rac2                     Started,STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.cvu
      1        ONLINE  ONLINE       rac1                     STABLE
ora.qosmserver
      1        ONLINE  ONLINE       rac2                     STABLE
ora.rac1.vip
      1        ONLINE  ONLINE       rac1                     STABLE
ora.rac2.vip
      1        ONLINE  ONLINE       rac2                     STABLE
ora.racdb.db
      1        ONLINE  ONLINE       rac2                     Open,HOME=/u01/app/o
                                                             racle/product/19c/db
                                                             _1,STABLE
      2        ONLINE  ONLINE       rac1                     Open,HOME=/u01/app/o
                                                             racle/product/19c/db
                                                             _1,STABLE
ora.scan1.vip
      1        ONLINE  ONLINE       rac2                     STABLE
ora.scan2.vip
      1        ONLINE  ONLINE       rac1                     STABLE
ora.scan3.vip
      1        ONLINE  ONLINE       rac1                     STABLE
--------------------------------------------------------------------------------

1. ocr  ,voting  dd 카피로 삭제

[root@rac1 disks]# dd if=/dev/zero of=/dev/oracleasm/disks/CRS1 bs=8192
dd: `/dev/oracleasm/disks/CRS1'에 쓰는 도중 오류 발생: 장치에 남은 공간이 없음
130945+0 records in
130944+0 records out
1072693248 bytes (1.1 GB) copied, 3.63769 s, 295 MB/s
[root@rac1 disks]# dd if=/dev/zero of=/dev/oracleasm/disks/CRS2 bs=8192
dd if=/dev/zero of=/dev/oracleasm/disks/CRS3 bs=8192
dd if=/dev/zero of=/dev/oracleasm/disks/CRS4 bs=8192dd: 
dd: `/dev/oracleasm/disks/CRS2'에 쓰는 도중 오류 발생: 장치에 남은 공간이 없음
130945+0 records in
130944+0 records out
1072693248 bytes (1.1 GB) copied, 2.12309 s, 505 MB/s
[root@rac1 disks]# dd if=/dev/zero of=/dev/oracleasm/disks/CRS3 bs=8192
dd: `/dev/oracleasm/disks/CRS3'에 쓰는 도중 오류 발생: 장치에 남은 공간이 없음
130945+0 records in
130944+0 records out
1072693248 bytes (1.1 GB) copied, 3.82352 s, 281 MB/s
[root@rac1 disks]# dd if=/dev/zero of=/dev/oracleasm/disks/CRS4 bs=8192
dd: `/dev/oracleasm/disks/CRS4'에 쓰는 도중 오류 발생: 장치에 남은 공간이 없음
130945+0 records in
130944+0 records out
1072693248 bytes (1.1 GB) copied, 4.25221 s, 252 MB/s

-- 비정상 동작 확인

[root@rac1 disks]# crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. OFFLINE  04d643e792984fa7bf2c85e2a9abcac2 (/dev/oracleasm/disks/CRS1) [CRS]
 2. PENDOFFL d3be744a62074f53bf7e41621a49fadc (/dev/oracleasm/disks/CRS2) [CRS]
 3. PENDOFFL d0469877eb0d4f82bf8c6a9c45193bc8 (/dev/oracleasm/disks/CRS3) [CRS]
Located 3 voting disk(s).
[root@rac1 disks]# ocrcheck
PROT-601: Failed to initialize ocrcheck
PROC-22: The OCR backend has an invalid format

2. CRS  정지

<GRID_HOME>/bin/crsctl stop crs -f

-- 그리드 정지 안 될때는 리 부팅 후 그리드 시작 초반에 다시 crsctl stop crs -f로 정지 해야 된다.

3. 구성 해제

De-configure the clusterware on the remote nodes (typically 'node2'-'noden')
 On all remote nodes, as root on Unix platforms and as the (current) Grid Infrastructure software owner on Windows, execute the following:

Unix example:
# <GRID_HOME>/crs/install/rootcrs.pl -deconfig -force -verbose 

[root@rac2 install]# ./rootcrs.pl -deconfig -force -verbose
./rootcrs.pl: line 88: =head1: command not found
./rootcrs.pl: line 90: rootcrs.pl: command not found
./rootcrs.pl: line 92: =head1: command not found
./rootcrs.pl: line 94: rootcrs.pl: command not found
./rootcrs.pl: line 95: parameter-file: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 96: [-deconfig: command not found
./rootcrs.pl: line 96: file: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 97: path: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 98: path: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 99: [[-prepatch: command not found
./rootcrs.pl: line 99: path: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 100: node: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 101: [-init]: command not found
./rootcrs.pl: line 102: path: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 103: sites: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 104: conversion: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 104: -createmc: command not found
./rootcrs.pl: line 104: -rollbacksc: command not found
./rootcrs.pl: line 104: dsc: 그런 파일이나 디렉터리가 없습니다
./rootcrs.pl: line 104: -convert: command not found
./rootcrs.pl: line 106: Options:: command not found
./rootcrs.pl: line 107: -verbose: command not found
./rootcrs.pl: line 108: -upgrade: command not found
./rootcrs.pl: line 109: -lock: command not found
./rootcrs.pl: line 110: -paramfile: command not found
./rootcrs.pl: line 111: -lastnode: command not found

wtmp begins Sun Jan  2 01:04:41 2022
./rootcrs.pl: line 113: with: command not found
./rootcrs.pl: line 114: -downgrade: command not found
./rootcrs.pl: line 115: -deconfig: command not found
./rootcrs.pl: line 116: -deinstall: command not found
./rootcrs.pl: line 117: -keepdg: command not found
./rootcrs.pl: line 118: -force: command not found
./rootcrs.pl: line 119: verified: command not found
./rootcrs.pl: line 120: -unlock: command not found
./rootcrs.pl: line 121: -crshome: command not found
./rootcrs.pl: line 122: syntax error near unexpected token `('
./rootcrs.pl: line 122: `   -prepatch   Perform required steps before the Oracle patching tool (Opatch) is invoked'

4) On the local node (typically 'node1'), as root on Unix platforms and as the (current) Grid Infrastructure software owner on Windows, execute the following:
a)  ONLY EXECUTE THE FOLLOWING COMMAND IF YOUR OCR and VD files ARE CONTAINED WITHIN THEIR OWN DISKGROUP:

Unix example:
# <GRID_HOME>/crs/install/rootcrs.pl -deconfig -force -verbose -lastnode

4. 그리드 인프라스트럭처 삭제

-- 1번 노드

[racdb1:/u01/app/19c/grid/deinstall]> ./deinstall
Checking for required files and bootstrapping ...
Please wait ...
로그 위치 /u01/app/oraInventory/logs/

############ ORACLE DECONFIG TOOL START ############


######################### DECONFIG CHECK OPERATION START #########################
## [시작] 설치 확인 구성 ##


 Oracle Clusterware 프로세스가 로컬 노드에서 실행 중이지 않으므로 설치 해제 유틸리티가 해당 홈이 구성된 노드 목록을 확인할 수 없습니다.

 다른 노드에 구성된 홈입니까(y - 예, n - 아니오)?[n]:

Oracle 홈 위치의 존재 여부를 확인하는 중/u01/app/19c/grid
설치 해제를 위해 선택된 Oracle 홈 유형: 클러스터용 Oracle Grid Infrastructure
설치 해제를 위해 선택된 Oracle 기본 위치: /u01/app/oracle
중앙 인벤토리 위치의 존재 여부를 확인하는 중/u01/app/oraInventory
Oracle Grid Infrastructure 홈의 존재 여부를 확인하는 중/u01/app/19c/grid
이 클러스터에 속한 노드: rac1
노드에 충분한 임시 공간을 사용할 수 있는지 확인하는 중: 'rac1'

## [종료] 설치 확인 구성 ##

Traces log file: /u01/app/oraInventory/logs//crsdc_2024-02-06_10-50-11-오전.log

네트워크 구성 확인 구성 시작

네트워크 구성 해제 추적 파일 위치: /u01/app/oraInventory/logs/netdc_check2024-02-06_10-50-12AM.log

구성을 해제할 RAC 리스너(SCAN 리스너 포함 안함)를 모두 지정하십시오. 모두 선택 해제하려면 .(마침표)를 입력하십시오. [ASMNET1LSNR_ASM,LISTENER,LISTENER_SCAN3,LISTENER_SCAN2,LISTENER_SCAN1]:

네트워크 구성 확인 구성 종료

ASM 확인 구성 시작

ASM 구성 해제 추적 파일 위치: /u01/app/oraInventory/logs/asmcadc_check2024-02-06_10-50-12AM.log

이 Oracle 홈에서 ASM 구성이 감지되지 않았습니다. 이 Oracle 홈 (y|n)에서 ASM을 구성했습니까? [n]: y
이 Oracle 홈 /u01/app/19c/grid에서 ASM(Automatic Storage Management) 인스턴스가 감지되었습니다.
ASM 진단 대상: /u01/app/oracle
ASM 디스크 그룹: 
ASM 디스크 문자열: /dev/oracleasm/disks/*
디스크 그룹이 삭제되지 않습니다.
 기존 디스크 그룹을 보존하려는 경우 또는 감지된 정보가 올바르지 않을 경우 'y'를 입력하여 수정할 수 있습니다. 위 정보 (y|n)을(를) 수정하겠습니까? [n]: 
데이터베이스 확인 구성 시작

데이터베이스 구성 해제 추적 파일 위치: /u01/app/oraInventory/logs/databasedc_check2024-02-06_10-50-12AM.log

이 Grid Infrastructure 홈에서 Oracle Grid 관리 데이터베이스를 찾을 수 없습니다.

데이터베이스 확인 구성 종료

######################### DECONFIG CHECK OPERATION END #########################


####################### DECONFIG CHECK OPERATION SUMMARY #######################
Oracle Grid Infrastructure 홈: /u01/app/19c/grid
이 클러스터에 속한 노드: rac1
Oracle 홈 설치 해제가 수행될 클러스터 노드:rac1
설치 해제를 위해 선택된 Oracle 홈: /u01/app/19c/grid
등록된 Oracle 홈이 있는 인벤토리 위치: /u01/app/oraInventory
구성을 해제할 RAC 리스너: ASMNET1LSNR_ASM,LISTENER,LISTENER_SCAN3,LISTENER_SCAN2,LISTENER_SCAN1
이 Oracle 홈에서 ASM 인스턴스의 구성이 해제됩니다.
이 Grid Infrastructure 홈에서 Oracle Grid 관리 데이터베이스를 찾을 수 없습니다.
계속하겠습니까(y - 예, n - 아니오)? [n]: y
이 세션의 로그는 '/u01/app/oraInventory/logs/deinstall_deconfig2024-02-06_10-49-07-AM.out'에 기록됩니다.
이 세션의 모든 오류 메시지는 '/u01/app/oraInventory/logs/deinstall_deconfig2024-02-06_10-49-07-AM.err'에 기록됩니다.

######################## DECONFIG CLEAN OPERATION START ########################
데이터베이스 구성 해제 추적 파일 위치: /u01/app/oraInventory/logs/databasedc_clean2024-02-06_10-50-12AM.log
ASM 구성 해제 추적 파일 위치: /u01/app/oraInventory/logs/asmcadc_clean2024-02-06_10-50-12AM.log
ASM 정리 구성 시작
ASM 정리 구성 종료

네트워크 구성 정리 구성 시작

네트워크 구성 해제 추적 파일 위치: /u01/app/oraInventory/logs/netdc_clean2024-02-06_10-50-12AM.log

RAC 리스너의 구성을 해제하는 중: ASMNET1LSNR_ASM,LISTENER,LISTENER_SCAN3,LISTENER_SCAN2,LISTENER_SCAN1

리스너의 구성을 해제하는 중: ASMNET1LSNR_ASM
    리스너를 정지하는 중: ASMNET1LSNR_ASM
    경고: 리스너 정지를 실패했습니다. 리스너가 실행 중이 아닐 수 있습니다.
리스너의 구성이 성공적으로 해제되었습니다.

리스너의 구성을 해제하는 중: LISTENER
    리스너를 정지하는 중: LISTENER
    경고: 리스너 정지를 실패했습니다. 리스너가 실행 중이 아닐 수 있습니다.
리스너의 구성이 성공적으로 해제되었습니다.

리스너의 구성을 해제하는 중: LISTENER_SCAN3
    리스너를 정지하는 중: LISTENER_SCAN3
    경고: 리스너 정지를 실패했습니다. 리스너가 실행 중이 아닐 수 있습니다.
리스너의 구성이 성공적으로 해제되었습니다.

리스너의 구성을 해제하는 중: LISTENER_SCAN2
    리스너를 정지하는 중: LISTENER_SCAN2
    경고: 리스너 정지를 실패했습니다. 리스너가 실행 중이 아닐 수 있습니다.
리스너의 구성이 성공적으로 해제되었습니다.

리스너의 구성을 해제하는 중: LISTENER_SCAN1
    리스너를 정지하는 중: LISTENER_SCAN1
    경고: 리스너 정지를 실패했습니다. 리스너가 실행 중이 아닐 수 있습니다.
리스너의 구성이 성공적으로 해제되었습니다.

모든 노드에서 이름 지정 방법 구성 파일의 구성을 해제하는 중...
이름 지정 방법 구성 파일의 구성이 성공적으로 해제되었습니다.

모든 노드에서 로컬 네트 서비스 이름 구성 파일의 구성을 해제하는 중...
로컬 네트 서비스 이름 구성 파일의 구성이 성공적으로 해제되었습니다.

모든 노드에서 디렉토리 사용 구성 파일의 구성을 해제하는 중...
디렉토리 사용 구성 파일의 구성이 성공적으로 해제되었습니다.

모든 노드에서 백업 파일의 구성을 해제하는 중...
백업 파일의 구성이 성공적으로 해제되었습니다.

네트워크 구성이 성공적으로 정리되었습니다.

네트워크 구성 정리 구성 종료


---------------------------------------->

아래의 deconfig 명령은 모든 원격 노드에서 병렬로 실행할 수 있습니다. 모든 원격 노드에서 실행이 완료된 후 로컬 노드에서 명령을 실행하십시오.

다음 명령을 "rac1" 노드에서 루트 사용자 또는 관리자로 실행하십시오.

/u01/app/19c/grid/crs/install/rootcrs.sh -force  -deconfig -paramfile "/tmp/deinstall2024-02-06_10-48-43AM/response/deinstall_OraGI19Home1.rsp" -lastnode

위의 명령 실행을 완료한 후 Enter를 누르십시오.

<----------------------------------------

-- 1번 서버 새창에서 Enter 전에 실행 

[root@rac1 ~]# /u01/app/19c/grid/crs/install/rootcrs.sh -force  -deconfig -paramfile "/tmp/deinstall2024-02-06_10-48-43AM/response/deinstall_OraGI19Home1.rsp" -lastnode
Using configuration parameter file: /tmp/deinstall2024-02-06_10-48-43AM/response/deinstall_OraGI19Home1.rsp
The log of current session can be found at:
  /u01/app/oraInventory/logs/crsdeconfig_rac1_2024-02-06_10-51-57AM.log
PRCR-1070 : 리소스 ora.cvu이(가) 등록되었는지 여부 확인을 실패했습니다.

CRS-0184 : Cannot communicate with the CRS daemon.

2024/02/06 10:52:05 CLSRSC-180: An error occurred while executing the command '/u01/app/19c/grid/bin/srvctl stop cvu -f'
PRCR-1070 : 리소스 ora.cvu이(가) 등록되었는지 여부 확인을 실패했습니다.

CRS-0184 : CRS 데몬과 통신할 수 없습니다.

2024/02/06 10:52:06 CLSRSC-180: An error occurred while executing the command '/u01/app/19c/grid/bin/srvctl remove cvu -f'
PRCR-1068 : 리소스 질의를 실패했습니다.

CRS-0184 : CRS 데몬과 통신할 수 없습니다.

2024/02/06 10:52:06 CLSRSC-180: An error occurred while executing the command '/u01/app/19c/grid/bin/srvctl stop scan_listener -f'
PRCR-1068 : 리소스 질의를 실패했습니다.

CRS-0184 : CRS 데몬과 통신할 수 없습니다.

2024/02/06 10:52:07 CLSRSC-180: An error occurred while executing the command '/u01/app/19c/grid/bin/srvctl remove scan_listener -y -f'
PRCR-1068 : 리소스 질의를 실패했습니다.
CRS-0184 : CRS 데몬과 통신할 수 없습니다.
2024/02/06 10:52:07 CLSRSC-180: An error occurred while executing the command '/u01/app/19c/grid/bin/srvctl stop scan -f'
PRCR-1068 : 리소스 질의를 실패했습니다.
CRS-0184 : CRS 데몬과 통신할 수 없습니다.
2024/02/06 10:52:08 CLSRSC-180: An error occurred while executing the command '/u01/app/19c/grid/bin/srvctl remove scan -y -f'
PRCR-1070 : 리소스 ora.net1.network이(가) 등록되었는지 여부 확인을 실패했습니다.
CRS-0184 : CRS 데몬과 통신할 수 없습니다.
PRCR-1070 : 리소스 ora.helper이(가) 등록되었는지 여부 확인을 실패했습니다.
CRS-0184 : CRS 데몬과 통신할 수 없습니다.
PRCR-1070 : 리소스 ora.ons이(가) 등록되었는지 여부 확인을 실패했습니다.
CRS-0184 : CRS 데몬과 통신할 수 없습니다.

2024/02/06 10:52:09 CLSRSC-180: An error occurred while executing the command '/u01/app/19c/grid/bin/srvctl config nodeapps'

CRS-4123: Oracle 고가용성 서비스가 시작되었습니다.
CRS-2672: 'ora.evmd'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2672: 'ora.mdnsd'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2676: 'ora.mdnsd' 시작('rac1')이 성공했습니다.
CRS-2676: 'ora.evmd' 시작('rac1')이 성공했습니다.
CRS-2672: 'ora.gpnpd'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2676: 'ora.gpnpd' 시작('rac1')이 성공했습니다.
CRS-2672: 'ora.cssdmonitor'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2672: 'ora.gipcd'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2676: 'ora.cssdmonitor' 시작('rac1')이 성공했습니다.
CRS-2676: 'ora.gipcd' 시작('rac1')이 성공했습니다.
CRS-2672: 'ora.cssd'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2672: 'ora.diskmon'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2676: 'ora.diskmon' 시작('rac1')이 성공했습니다.
CRS-2676: 'ora.cssd' 시작('rac1')이 성공했습니다.
CRS-2672: 'ora.crf'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2672: 'ora.ctssd'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2672: 'ora.cluster_interconnect.haip'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2676: 'ora.crf' 시작('rac1')이 성공했습니다.
CRS-2676: 'ora.ctssd' 시작('rac1')이 성공했습니다.
CRS-2676: 'ora.cluster_interconnect.haip' 시작('rac1')이 성공했습니다.
CRS-2672: 'ora.asm'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
CRS-2676: 'ora.asm' 시작('rac1')이 성공했습니다.
CRS-2672: 'ora.storage'을(를) 'rac1'에서 시작하려고 시도하는 중입니다.
diskgroup CRS not mounted ()
CRS-5055: 클러스터에 실행 중인 ASM 인스턴스가 없으므로 ASM 인스턴스에 접속할 수 없습니다.
CRS-2674: 'ora.storage' 시작('rac1')이 실패했습니다.
CRS-2679: 'ora.storage'('rac1')을(를) 정리하려고 시도하는 중입니다.
CRS-2681: 'ora.storage' 정리('rac1')가 성공했습니다.
CRS-2673: 'ora.asm'을(를) 'rac1'에서 정지하려고 시도하는 중입니다.
CRS-2677: 'ora.asm' 정지('rac1')가 성공했습니다.
CRS-2673: 'ora.cluster_interconnect.haip'을(를) 'rac1'에서 정지하려고 시도하는 중입니다.
CRS-2677: 'ora.cluster_interconnect.haip' 정지('rac1')가 성공했습니다.
CRS-2673: 'ora.ctssd'을(를) 'rac1'에서 정지하려고 시도하는 중입니다.
CRS-2677: 'ora.ctssd' 정지('rac1')가 성공했습니다.
CRS-2673: 'ora.crf'을(를) 'rac1'에서 정지하려고 시도하는 중입니다.
CRS-2677: 'ora.crf' 정지('rac1')가 성공했습니다.
CRS-4000: Start 명령을 실패했거나 오류와 함께 완료되었습니다.
2024/02/06 11:03:31 CLSRSC-260: Clusterware exclusive mode start of Cluster Ready Services failed
2024/02/06 11:03:31 CLSRSC-558: failed to deconfigure ASM
2024/02/06 11:03:31 CLSRSC-651: One or more deconfiguration steps failed, but the deconfiguration process continued because the -force option was specified.
2024/02/06 11:03:49 CLSRSC-4006: Removing Oracle Trace File Analyzer (TFA) Collector.
2024/02/06 11:05:48 CLSRSC-4007: Successfully removed Oracle Trace File Analyzer (TFA) Collector.
2024/02/06 11:05:52 CLSRSC-336: Successfully deconfigured Oracle Clusterware stack on this node
2024/02/06 11:05:52 CLSRSC-559: Ensure that the GPnP profile data under the 'gpnp' directory in /u01/app/19c/grid is deleted on each node before using the software in the current Grid Infrastructure home for reconfiguration.

-- 2번 서버도 같은 방식으로 deinstall

5. OCR ,VOTING DISK 파일 ASM등록

[root@rac1 ~]# oracleasm createdisk CRS1 /dev/sdb1
Writing disk header: done
Instantiating disk: done

[root@rac1 ~]# oracleasm createdisk CRS 2 /dev/sdc1
Writing disk header: done
Instantiating disk: done

[root@rac1 ~]# oracleasm createdisk CRS 3 /dev/sdd1
Writing disk header: done
Instantiating disk: done

[root@rac1 ~]# oracleasm createdisk CRS4 /dev/sdh1
Writing disk header: done
Instantiating disk: done

[root@rac1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac1 ~]# oracleasm listdisks
ASM01
ASM02
ASM03
ASM04
ASM05
CRS1
CRS2
CRS3

CRS4
DATA
DATA2
FRA

6. 그리드 인프라스트럭처 재 설치

-- cvuqdisk 패키지 1, 2번 노드 설치

[root@rac1 ~]# cd $GRID_HOME/cv/rpm

[root@rac1 rpm]# ls -ltr
합계 12
-rw-r--r--. 1 oracle oinstall 11412  3월 13  2019 cvuqdisk-1.0.10-1.rpm
[root@rac1 rpm]# rpm -Uvh cvuqdisk-1.0.10-1.rpm 
준비 중...                         ################################# [100%]
Updating / installing...
   1:cvuqdisk-1.0.10-1                ################################# [100%]

-- SCP 로 2번 노드로 복사 후 설치

scp cvuqdisk-1.0.10-1.rpm root@rac2:/tmp/

-- 2번 서버에서 설치

[root@rac2 tmp]# rpm -Uvh cvuqdisk-1.0.10-1.rpm 
준비 중...                         ################################# [100%]
Updating / installing...
   1:cvuqdisk-1.0.10-1                ################################# [100%]

-- 처음에 설치 했던 대로 설치

cd $GRID_HOME
./gridSetup.sh

-- 설치 중  ASM 볼륨(OCR,Voting disk ) 설정

-- #1번 서버 설치 중 root.sh 실행하라고 나오면 1.서버 root.sh실행 후 2번서버root.sh실행

[root@rac1 grid]# ./root.sh
Performing root user operation.

The following environment variables are set as:
    ORACLE_OWNER= oracle
    ORACLE_HOME=  /u01/app/19c/grid

Enter the full pathname of the local bin directory: [/usr/local/bin]: 
The contents of "dbhome" have not changed. No need to overwrite.
The contents of "oraenv" have not changed. No need to overwrite.
The contents of "coraenv" have not changed. No need to overwrite.

Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Relinking oracle with rac_on option
Using configuration parameter file: /u01/app/19c/grid/crs/install/crsconfig_params
The log of current session can be found at:
  /u01/app/oracle/crsdata/rac2/crsconfig/rootcrs_rac2_2024-02-06_02-50-47PM.log
2024/02/06 14:50:56 CLSRSC-594: Executing installation step 1 of 19: 'SetupTFA'.
2024/02/06 14:50:56 CLSRSC-594: Executing installation step 2 of 19: 'ValidateEnv'.
2024/02/06 14:50:57 CLSRSC-594: Executing installation step 3 of 19: 'CheckFirstNode'.
2024/02/06 14:50:58 CLSRSC-594: Executing installation step 4 of 19: 'GenSiteGUIDs'.
2024/02/06 14:50:58 CLSRSC-594: Executing installation step 5 of 19: 'SetupOSD'.
2024/02/06 14:50:58 CLSRSC-594: Executing installation step 6 of 19: 'CheckCRSConfig'.
2024/02/06 14:50:59 CLSRSC-594: Executing installation step 7 of 19: 'SetupLocalGPNP'.
2024/02/06 14:51:01 CLSRSC-594: Executing installation step 8 of 19: 'CreateRootCert'.
2024/02/06 14:51:01 CLSRSC-594: Executing installation step 9 of 19: 'ConfigOLR'.
2024/02/06 14:51:10 CLSRSC-594: Executing installation step 10 of 19: 'ConfigCHMOS'.
2024/02/06 14:51:10 CLSRSC-594: Executing installation step 11 of 19: 'CreateOHASD'.
2024/02/06 14:51:11 CLSRSC-594: Executing installation step 12 of 19: 'ConfigOHASD'.
2024/02/06 14:51:12 CLSRSC-330: Adding Clusterware entries to file 'oracle-ohasd.service'
2024/02/06 14:51:32 CLSRSC-4002: Successfully installed Oracle Trace File Analyzer (TFA) Collector.
2024/02/06 14:51:41 CLSRSC-594: Executing installation step 13 of 19: 'InstallAFD'.
2024/02/06 14:51:43 CLSRSC-594: Executing installation step 14 of 19: 'InstallACFS'.
2024/02/06 14:51:44 CLSRSC-594: Executing installation step 15 of 19: 'InstallKA'.
2024/02/06 14:51:46 CLSRSC-594: Executing installation step 16 of 19: 'InitConfig'.
2024/02/06 14:51:54 CLSRSC-594: Executing installation step 17 of 19: 'StartCluster'.
2024/02/06 14:52:50 CLSRSC-343: Successfully started Oracle Clusterware stack
2024/02/06 14:52:50 CLSRSC-594: Executing installation step 18 of 19: 'ConfigNode'.
2024/02/06 14:53:08 CLSRSC-594: Executing installation step 19 of 19: 'PostConfig'.
2024/02/06 14:53:16 CLSRSC-325: Configure Oracle Grid Infrastructure for a Cluster ... succeeded

7. root.sh 완료 후 제대로 실행 되었는 지 확인

Unix: <GRID_HOME>/bin/crsctl check cluster -all
[root@rac2 grid]# crsctl check cluster -all
**************************************************************
rac1:
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************
rac2:
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************

For Grid Infrastructure Standalone: <GRID_HOME>/bin/crsctl check has

For Grid Infrastructure Standalone, it's possible you will need to re-create your ora.asm resource using the following commands:

Unix:

srvctl add asm
srvctl start asm

8. 기존 asm 디스크 그룹 마운트

-- asmca 실행 후 기존 디스크 그룹 마운트

   모두 마운트 선택

---------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMDATA.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        ONLINE  OFFLINE                               STABLE
ora.CRS.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.DATA.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.FRA.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE

9 DB 및 인스턴스 정보 등록

DB 등록

[racdb1:/home/oracle]> srvctl add database -d racdb -o $ORACLE_HOME

인스턴스 정보 등록

[racdb1:/home/oracle]> srvctl add instance -d racdb  -i racdb1 -n rac1
[racdb1:/home/oracle]> srvctl add instance -d racdb  -i racdb2 -n rac2

10  오라클 기동 

[racdb1:/home/oracle]> srvctl start database -d racdb
[racdb1:/home/oracle]> crsctl stat res -t
--------------------------------------------------------------------------------
Name           Target  State        Server                   State details       
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
ora.chad
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
ora.net1.network
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
ora.ons
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMDATA.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        ONLINE  OFFLINE                               STABLE
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        ONLINE  OFFLINE                               STABLE
ora.CRS.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.DATA.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.FRA.dg(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       rac1                     STABLE
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       rac2                     STABLE
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       rac2                     STABLE
ora.asm(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     Started,STABLE
      2        ONLINE  ONLINE       rac2                     Started,STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
      1        ONLINE  ONLINE       rac1                     STABLE
      2        ONLINE  ONLINE       rac2                     STABLE
      3        OFFLINE OFFLINE                               STABLE
ora.cvu
      1        ONLINE  ONLINE       rac2                     STABLE
ora.qosmserver
      1        ONLINE  ONLINE       rac2                     STABLE
ora.rac1.vip
      1        ONLINE  ONLINE       rac1                     STABLE
ora.rac2.vip
      1        ONLINE  ONLINE       rac2                     STABLE
ora.racdb.db
      1        ONLINE  ONLINE       rac1                     Open,HOME=/u01/app/o
                                                             racle/product/19c/db
                                                             _1,STABLE
      2        ONLINE  ONLINE       rac2                     Open,HOME=/u01/app/o
                                                             racle/product/19c/db
                                                             _1,STABLE
ora.scan1.vip
      1        ONLINE  ONLINE       rac1                     STABLE
ora.scan2.vip
      1        ONLINE  ONLINE       rac2                     STABLE
ora.scan3.vip
      1        ONLINE  ONLINE       rac2                     STABLE
--------------------------------------------------------------------------------

완료!!!