# PaCkAgE DaTaStReAm JAVAtnfd 1 0 # end of header 0707010005ae40000081a4000039220000000a000000014875eae2000000e70000016d0000089b00000000000000000000001100000000JAVAtnfd/pkginfoVERSION=1.0 PKG=JAVAtnfd NAME=TNF debugging scripts ARCH=sparc.sun4u,sparc.sun4v CATEGORY=application VENDOR=Sun Microsystems - TSC-SYSTEMS BASEDIR=/opt/JAVAtnfd CLASSES=none 58 EMAIL=tim.uglow@sun.com PSTAMP=enoexec20080710115634 0707010005ae1d000081a4000039220000000a000000014875eae2000001980000016d0000089b00000000000000000000001000000000JAVAtnfd/pkgmap: 1 0 1 d none bin 0755 root sys 1 f none bin/tnfscript 0755 root sys 3005 57138 1215687393 1 d none data 0777 root sys 1 d none docs 0755 root sys 1 f none docs/README.txt 0444 root sys 3989 10488 1215687393 1 i pkginfo 231 18746 1215687394 1 i postinstall 201 17053 1215079940 1 i postremove 189 16035 1215080023 1 i preinstall 19 1353 1215080063 1 i request 15 1097 1215079748 1 i space 10 635 1215082575 07070100000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000b00000000TRAILER!!!0707010005ae40000081a4000039220000000a000000014875eae2000000e70000016d0000089b00000000000000000000000800000000pkginfoVERSION=1.0 PKG=JAVAtnfd NAME=TNF debugging scripts ARCH=sparc.sun4u,sparc.sun4v CATEGORY=application VENDOR=Sun Microsystems - TSC-SYSTEMS BASEDIR=/opt/JAVAtnfd CLASSES=none 58 EMAIL=tim.uglow@sun.com PSTAMP=enoexec20080710115634 0707010005ae1d000081a4000039220000000a000000014875eae2000001980000016d0000089b00000000000000000000000700000000pkgmap: 1 0 1 d none bin 0755 root sys 1 f none bin/tnfscript 0755 root sys 3005 57138 1215687393 1 d none data 0777 root sys 1 d none docs 0755 root sys 1 f none docs/README.txt 0444 root sys 3989 10488 1215687393 1 i pkginfo 231 18746 1215687394 1 i postinstall 201 17053 1215079940 1 i postremove 189 16035 1215080023 1 i preinstall 19 1353 1215080063 1 i request 15 1097 1215079748 1 i space 10 635 1215082575 0707010005ae4a000041ed000039220000000a000000024875eae3000000000000016d0000089b00000000000000000000000800000000install0707010005ae4f000081a4000039220000000a00000001486cb04f0000000a0000016d0000089b00000000000000000000000e00000000install/spacedata 20 5 0707010005ae4e000081a4000039220000000a00000001486ca5440000000f0000016d0000089b00000000000000000000001000000000install/requestCLASSES="none" 0707010005ae4c000081a4000039220000000a00000001486ca657000000bd0000016d0000089b00000000000000000000001300000000install/postremove#!/bin/ksh echo "about to rem_drv and then add_drv the tnf driver to reset" echo "the permissions so only root can run prex -k" rem_drv tnf add_drv -m '* 0600 root sys' tnf echo "done" 0707010005ae4d000081a4000039220000000a00000001486ca67f000000130000016d0000089b00000000000000000000001300000000install/preinstall#!/bin/ksh exit 0 0707010005ae4b000081a4000039220000000a00000001486ca604000000c90000016d0000089b00000000000000000000001400000000install/postinstall#!/bin/ksh echo "about to rem_drv and then add_drv the tnf driver to open" echo "the permissions so oracle can use prex -k as user oracle." rem_drv tnf add_drv -m '* 0666 root sys' tnf echo "done" 0707010005ae41000041ed000039220000000a000000044875eae3000000000000016d0000089b00000000000000000000000600000000reloc0707010005ae48000041ed000039220000000a000000024875eae3000000000000016d0000089b00000000000000000000000b00000000reloc/docs0707010005ae4900008124000039220000000a000000014875eae100000f950000016d0000089b00000000000000000000001600000000reloc/docs/README.txt To allow oracle processes running as user "oracle" to stop the tnf kernel tracing we need to change the permissions on the tnf driver. Currently you have .. # ls -alL /dev/tnf* crw------- 1 root sys 114, 0 Apr 10 22:02 /dev/tnfctl crw------- 1 root sys 114, 1 Apr 10 22:02 /dev/tnfmap and we need to change it to ... # ls -alL /dev/tnf* crw-rw-rw- 1 root sys 114, 0 Jun 27 15:57 /dev/tnfctl crw-rw-rw- 1 root sys 114, 1 Jun 27 15:57 /dev/tnfmap This is done by the following two commands... # rem_drv tnf # add_drv -m '* 0666 root sys' tnf the install script for JAVAtnfd package will do this for you and the uninstall script will put it back to its original state This change would allow ordinary users to affect the kernel tracing but most users don't know about the subsystem and the only information they can gather is kernel events, there is no actual data visible. Installing the scripts. ======================= 1) copy JAVAtnfd.datastream.pkg to /tmp on your machine 2) pkgadd -d /tmp/JAVAtnfd.datastream.pkg You may get warning messages like .. "Device busy Cannot unload module: tnf Will be unloaded upon reboot." These can be ingnored.. devfsadm will have changed the permissions. To start the tnf tracing.. as root please run ... # cd /opt/JAVAtnfd/data # /usr/bin/nohup /opt/JAVAtnfd/bin/tnfscript & then check in /opt/JAVAtnfd/data/nohup.out for errors, every time you start it it will append to nohup.out. If you needed to stop the tracing .. looking in /opt/JAVAtnfd/data/nohup.out for the last line like .. "tnfscript pid XXXX starting" and then run ... # kill XXXX should stop it. When the oracle debug executes it will touch /opt/JAVAtnfd/data/stop and stop the tnf tracing and that will cause the tnfscript to exit at the end of its 10 minute loop - look at the end of /opt/JAVAtnfd/data/nohup.out for "tnfscript pid XXXX exiting as requested by Oracle". Before it exits it will extract the required files into /opt/JAVAtnfd/data/*.'date' and email root the location of the four files that need to be sent to support together with the Oracle alert log and the trace file that will be mentioned in the alert.log file, something like.... Errors in file ORA-7445... is also required. Instructions from Oracle to get Oracle processes to stop the tnf on a ORA-7445 error. -------------------------------------------------- 1) in init.ora define the following 2 parameters event="7445 debug" _oradbg_pathname=<$ORACLE_HOME>/bin/oradbg where $ORACLE_HOME is the expanded path of the $ORACLE_HOME env var 2) in $ORACLE_HOME/bin, place a script called oradbg containing: #!/bin/sh echo "stopping TNF collection using prex" >> /tmp/oradbg.$$ echo "ktrace off" | /usr/bin/prex -k echo $$ > /opt/JAVAtnfd/data/stop echo `/bin/date` > /tmp/oradbg.$$ echo "done\." >> /tmp/oradbg.$$ 3) add execute to the oradbg script 4) restart the instance Upon a 7445 error being encountered (whereby oracle's signal handler spots a sigsegv/core dump) we will disable the prex via the built-in oracle 'debug' event and a file will be written to /tmp containing the date and time and a note that we stopped prex. It willalso touch the file /opt/JAVAtnfd/data/stop which will signal the Sun tnf script to stop. Obviously if the customer has any path differences for any of the bins they should modify accordingly. This can easily be tested after restarting the instance by running: sqlplus / alter session set events '942 debug'; select * from ; This will generate an error along the lines: ERROR at line 1: ORA-00942: table or view does not exist and the debug event will fire on the temporary session-level dummy version of the event and will run the prex command. They can then check /tmp for a new file with the details called oradbg. --------------------------------------------- 0707010005ae46000041ed000039220000000a000000024875eae2000000000000016d0000089b00000000000000000000000a00000000reloc/bin0707010005ae47000081ed000039220000000a000000014875eae100000bbd0000016d0000089b00000000000000000000001400000000reloc/bin/tnfscript#!/bin/ksh cd /opt/JAVAtnfd/data echo "this script will every 10 minutes stop the tnf tracing, drop the" echo "buffer , reallocate it and start the tracing again. This has to be" echo "done else the buffer becomes confused and space is lost, there is" echo "a chance that the problem will happen as the tracing is stopped and" echo "the event will be missed." echo "" echo "the oracle diagnostic scrip that is invoked will" echo "touch /opt/JAVAtnfd/data/stop and that will tell this script to stop" echo "tnfscript pid $$ starting" echo "" if [ -f /opt/JAVAtnfd/data/stop ] then echo "tnfscript pid $$ starting but /opt/JAVAtnfd/data/stop exists !!" echo "please check the Oracle alert logs" echo "To restart tracing rm /opt/JAVAtnfd/data/stop and " echo "try this command again." exit 0 fi echo "initial tnf setup" ( echo "buffer alloc 128m" echo "enable \$all" echo "trace \$all" echo "ktrace on" ) | /usr/bin/prex -k echo "starting a 10 minute loop" while sleep 600 do date if [ -f /opt/JAVAtnfd/data/stop ] then echo "tnfscript pid $$ exiting as request by Oracle - /opt/JAVAtnfd/data/stop" orapid=$(cat /opt/JAVAtnfd/data/stop) now=$(date +%y%m%d%H%M%S) echo "the oracle pid of interest is $orapid" echo "about to extract the kernel trace file to /opt/JAVAtnfd/data/tnf.out.${now}" echo " emailing instructions to root" /usr/bin/tnfxtract /opt/JAVAtnfd/data/tnf.out.${now} cp /dev/ksyms /opt/JAVAtnfd/data/ksyms.${now} cp /etc/name_to_sysnum /opt/JAVAtnfd/data/name_to_sysnum.${now} cp /tmp/oradbg.${orapid} /opt/JAVAtnfd/data/oradbg.${orapid}.${now} /usr/bin/mailx -s "tnf tracing stopped at ${now}" root <<-MAILEOF tnfscript pid $$ stopping at ${now}. please ship Oracle alert file and $(ls -alt /opt/JAVAtnfd/data/*${now}) to support for analysis. Please also include the oracle trace file referenced in the Oracle alert.log for this ORA-7445 event. MAILEOF exit 1 fi # we willalso check the ktrace state, if off assume oracle stopped it. state=$(echo "ktrace" | prex -k | (read a ; read a b c; echo $c)) if [ aa${state} == aa"off" ] then echo "tnfscript pid $$ exiting as request by Oracle - ktrace is off" orapid=$(cat /opt/JAVAtnfd/data/stop) now=$(date +%y%m%d%H%M%S) echo "about to extract the kernel trace file to /opt/JAVAtnfd/data/tnf.out.${now}" echo " emailing instructions to root" /usr/bin/tnfxtract /opt/JAVAtnfd/data/tnf.out.${now} cp /dev/ksyms /opt/JAVAtnfd/data/ksyms.${now} cp /etc/name_to_sysnum /opt/JAVAtnfd/data/name_to_sysnum.${now} cp /tmp/oradbg.${orapid} /opt/JAVAtnfd/data/oradbg.${orapid}.${now} /usr/bin/mailx -s "tnf tracing stopped at ${now}" root <<-MAILEOF tnfscript pid $$ stopping at ${now}. please ship Oracle alert file and $(ls -alt /opt/JAVAtnfd/data/*${now}) to support for analysis. MAILEOF exit 1 fi ( echo "ktrace off" echo "buffer dealloc" echo "buffer alloc 128m" echo "enable \$all" echo "trace \$all" echo "ktrace on" ) | /usr/bin/prex -k done 07070100000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000b00000000TRAILER!!!