Enable additional meta keys in table-map-custom.xml for enhanced log information in the NetWitness Platform

Issue

Not all meta keys are enabled in the table-map.xml file which means some information from log parsers may not be captured. Not all this information may be required, but this script is a way to highlight these additional meta keys so that they can be added into the following file.

 /etc/netwitness/ng/envision/etc/table-map-custom.xml
 

The script attached to this solution displays the meta keys to add to this file. The script highlights meta keys that are:

Set to transient in the table-map.xml file
ExtensionKeys in the CEF parser

Resolution

Copy the attached script to the Log Decoder appliance and make it executable with the command below.

 chmod +x findmissing.sh

On running the script the output below will be displayed.

  ./findmissing.sh
 
Additional Meta keys for table-map-custom.xml can be found in /tmp/TOADD.txt
 
Paste the contents of this file between the <mappings> </mappings> tags
 
into the file /etc/netwitness/ng/envision/etc/table-map-custom.xml

An example of the output in /tmp/TOADD.txt is shown below.

 <!-- BEGIN List of keys Not in table-map-custom.xml -->
 
<mapping envisionName="cn_acttimeout" nwName="cn_acttimeout" flags="None"/>
 
<mapping envisionName="cn_asn_dst" nwName="cn_asn_dst" flags="None"/>
 
<mapping envisionName="cn_asn_src" nwName="cn_asn_src" flags="None"/>
 
<mapping envisionName="cn_bgpv4nxthop" nwName="cn_bgpv4nxthop" flags="None"/>
 
<mapping envisionName="cn_ctr_dst_code" nwName="cn_ctr_dst_code" flags="None"/>
 
<mapping envisionName="cn_dst_tos" nwName="cn_dst_tos" flags="None"/>
 
<mapping envisionName="cn_dst_vlan" nwName="cn_dst_vlan" flags="None"/>
 
<mapping envisionName="cn_engine_id" nwName="cn_engine_id" flags="None"/>
 
<mapping envisionName="cn_engine_type" nwName="cn_engine_type" flags="None"/>
 
<mapping envisionName="cn_eventver" nwName="cn_eventver" flags="None"/>
 
<mapping envisionName="cn_f_switch" nwName="cn_f_switch" flags="None"/>
 
<mapping envisionName="cn_fld" nwName="cn_fld" flags="None"/>
 
<mapping envisionName="cn_flowsampid" nwName="cn_flowsampid" flags="None"/>
 
<mapping envisionName="cn_flowsampintv" nwName="cn_flowsampintv" flags="None"/>
 
<mapping envisionName="cn_flowsampmode" nwName="cn_flowsampmode" flags="None"/>
 
<mapping envisionName="cn_inacttimeout" nwName="cn_inacttimeout" flags="None"/>
 
<mapping envisionName="cn_inpermbyts" nwName="cn_inpermbyts" flags="None"/>
 
<mapping envisionName="cn_inpermpckts" nwName="cn_inpermpckts" flags="None"/>
 
<mapping envisionName="cn_invalid" nwName="cn_invalid" flags="None"/>
 
<mapping envisionName="cn_ip_proto_ver" nwName="cn_ip_proto_ver" flags="None"/>
 
<mapping envisionName="cn_ipv4_ident" nwName="cn_ipv4_ident" flags="None"/>
 
<mapping envisionName="cn_l_switch" nwName="cn_l_switch" flags="None"/>
 
<mapping envisionName="cn_log_did" nwName="cn_log_did" flags="None"/>
 
<mapping envisionName="cn_log_rid" nwName="cn_log_rid" flags="None"/>
 
<mapping envisionName="cn_max_ttl" nwName="cn_max_ttl" flags="None"/>
 
<mapping envisionName="cn_maxpcktlen" nwName="cn_maxpcktlen" flags="None"/>
 
<mapping envisionName="cn_min_ttl" nwName="cn_min_ttl" flags="None"/>
 
<mapping envisionName="cn_minpcktlen" nwName="cn_minpcktlen" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_1" nwName="cn_mpls_lbl_1" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_10" nwName="cn_mpls_lbl_10" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_2" nwName="cn_mpls_lbl_2" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_3" nwName="cn_mpls_lbl_3" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_4" nwName="cn_mpls_lbl_4" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_5" nwName="cn_mpls_lbl_5" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_6" nwName="cn_mpls_lbl_6" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_7" nwName="cn_mpls_lbl_7" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_8" nwName="cn_mpls_lbl_8" flags="None"/>
 
<mapping envisionName="cn_mpls_lbl_9" nwName="cn_mpls_lbl_9" flags="None"/>
 
<mapping envisionName="cn_mplstoplabel" nwName="cn_mplstoplabel" flags="None"/>
 
<mapping envisionName="cn_mplstoplabip" nwName="cn_mplstoplabip" flags="None"/>
 
<mapping envisionName="cn_mul_dst_byt" nwName="cn_mul_dst_byt" flags="None"/>
 
<mapping envisionName="cn_mul_dst_pks" nwName="cn_mul_dst_pks" flags="None"/>
 
<mapping envisionName="cn_muligmptype" nwName="cn_muligmptype" flags="None"/>
 
<mapping envisionName="cn_oldfileid" nwName="cn_oldfileid" flags="None"/>
 
<mapping envisionName="cn_oldfilesize" nwName="cn_oldfilesize" flags="None"/>
 
<mapping envisionName="cn_rpackets" nwName="cn_rpackets" flags="None"/>
 
<mapping envisionName="cn_sampalgo" nwName="cn_sampalgo" flags="None"/>
 
<mapping envisionName="cn_sampint" nwName="cn_sampint" flags="None"/>
 
<mapping envisionName="cn_seqctr" nwName="cn_seqctr" flags="None"/>
 
<mapping envisionName="cn_spackets" nwName="cn_spackets" flags="None"/>
 
<mapping envisionName="cn_src_tos" nwName="cn_src_tos" flags="None"/>
 
<mapping envisionName="cn_src_vlan" nwName="cn_src_vlan" flags="None"/>
 
<mapping envisionName="cn_sysuptime" nwName="cn_sysuptime" flags="None"/>
 
<mapping envisionName="cn_template_id" nwName="cn_template_id" flags="None"/>
 
<mapping envisionName="cn_totbytsexp" nwName="cn_totbytsexp" flags="None"/>
 
<mapping envisionName="cn_totflowexp" nwName="cn_totflowexp" flags="None"/>
 
<mapping envisionName="cn_totpcktsexp" nwName="cn_totpcktsexp" flags="None"/>
 
<mapping envisionName="cn_unixnanosecs" nwName="cn_unixnanosecs" flags="None"/>
 
<mapping envisionName="cn_v6flowlabel" nwName="cn_v6flowlabel" flags="None"/>
 
<mapping envisionName="cn_v6optheaders" nwName="cn_v6optheaders" flags="None"/>
 
<mapping envisionName="cs_accesskeyid" nwName="cs_accesskeyid" flags="None"/>
 
<mapping envisionName="cs_accountid" nwName="cs_accountid" flags="None"/>
 
<mapping envisionName="cs_agency_dst" nwName="cs_agency_dst" flags="None"/>
 
<mapping envisionName="cs_analyzedby" nwName="cs_analyzedby" flags="None"/>
 
<mapping envisionName="cs_av_other" nwName="cs_av_other" flags="None"/>
 
<mapping envisionName="cs_av_primary" nwName="cs_av_primary" flags="None"/>
 
<mapping envisionName="cs_av_secondary" nwName="cs_av_secondary" flags="None"/>
 
<mapping envisionName="cs_bgpv6nxthop" nwName="cs_bgpv6nxthop" flags="None"/>
 
<mapping envisionName="cs_customdate" nwName="cs_customdate" flags="None"/>
 
<mapping envisionName="cs_datecret" nwName="cs_datecret" flags="None"/>
 
<mapping envisionName="cs_devfacility" nwName="cs_devfacility" flags="None"/>
 
<mapping envisionName="cs_devservice" nwName="cs_devservice" flags="None"/>
 
<mapping envisionName="cs_dst_tld" nwName="cs_dst_tld" flags="None"/>
 
<mapping envisionName="cs_eth_dst_ven" nwName="cs_eth_dst_ven" flags="None"/>
 
<mapping envisionName="cs_eth_src_ven" nwName="cs_eth_src_ven" flags="None"/>
 
<mapping envisionName="cs_event_uuid" nwName="cs_event_uuid" flags="None"/>
 
<mapping envisionName="cs_filectime" nwName="cs_filectime" flags="None"/>
 
<mapping envisionName="cs_fileid" nwName="cs_fileid" flags="None"/>
 
<mapping envisionName="cs_filemtime" nwName="cs_filemtime" flags="None"/>
 
<mapping envisionName="cs_fileperm" nwName="cs_fileperm" flags="None"/>
 
<mapping envisionName="cs_fld" nwName="cs_fld" flags="None"/>
 
<mapping envisionName="cs_frametype" nwName="cs_frametype" flags="None"/>
 
<mapping envisionName="cs_identityarn" nwName="cs_identityarn" flags="None"/>
 
<mapping envisionName="cs_if_desc" nwName="cs_if_desc" flags="None"/>
 
<mapping envisionName="cs_if_name" nwName="cs_if_name" flags="None"/>
 
<mapping envisionName="cs_ip_next_hop" nwName="cs_ip_next_hop" flags="None"/>
 
<mapping envisionName="cs_ipv4dstpre" nwName="cs_ipv4dstpre" flags="None"/>
 
<mapping envisionName="cs_ipv4srcpre" nwName="cs_ipv4srcpre" flags="None"/>
 
<mapping envisionName="cs_lifetime" nwName="cs_lifetime" flags="None"/>
 
<mapping envisionName="cs_log_medium" nwName="cs_log_medium" flags="None"/>
 
<mapping envisionName="cs_loginname" nwName="cs_loginname" flags="None"/>
 
<mapping envisionName="cs_oldfilectime" nwName="cs_oldfilectime" flags="None"/>
 
<mapping envisionName="cs_oldfilehash" nwName="cs_oldfilehash" flags="None"/>
 
<mapping envisionName="cs_oldfilemtime" nwName="cs_oldfilemtime" flags="None"/>
 
<mapping envisionName="cs_oldfilename" nwName="cs_oldfilename" flags="None"/>
 
<mapping envisionName="cs_oldfilepath" nwName="cs_oldfilepath" flags="None"/>
 
<mapping envisionName="cs_oldfileperm" nwName="cs_oldfileperm" flags="None"/>
 
<mapping envisionName="cs_oldfiletype" nwName="cs_oldfiletype" flags="None"/>
 
<mapping envisionName="cs_operation" nwName="cs_operation" flags="None"/>
 
<mapping envisionName="cs_packettype" nwName="cs_packettype" flags="None"/>
 
<mapping envisionName="cs_paramkey" nwName="cs_paramkey" flags="None"/>
 
<mapping envisionName="cs_paramvalue" nwName="cs_paramvalue" flags="None"/>
 
<mapping envisionName="cs_payload" nwName="cs_payload" flags="None"/>
 
<mapping envisionName="cs_registrant" nwName="cs_registrant" flags="None"/>
 
<mapping envisionName="cs_registrar" nwName="cs_registrar" flags="None"/>
 
<mapping envisionName="cs_req_inst_id" nwName="cs_req_inst_id" flags="None"/>
 
<mapping envisionName="cs_reqcookies" nwName="cs_reqcookies" flags="None"/>
 
<mapping envisionName="cs_reqid" nwName="cs_reqid" flags="None"/>
 
<mapping envisionName="cs_resp_acctid" nwName="cs_resp_acctid" flags="None"/>
 
<mapping envisionName="cs_rpayload" nwName="cs_rpayload" flags="None"/>
 
<mapping envisionName="cs_sampler_name" nwName="cs_sampler_name" flags="None"/>
 
<mapping envisionName="cs_streams" nwName="cs_streams" flags="None"/>
 
<mapping envisionName="cs_tenant" nwName="cs_tenant" flags="None"/>
 
<mapping envisionName="cs_tenantid" nwName="cs_tenantid" flags="None"/>
 
<mapping envisionName="cs_transaction" nwName="cs_transaction" flags="None"/>
 
<mapping envisionName="cs_user" nwName="cs_user" flags="None"/>
 
<mapping envisionName="cs_v6nxthop" nwName="cs_v6nxthop" flags="None"/>
 
<mapping envisionName="cs_whois_server" nwName="cs_whois_server" flags="None"/>
 
<mapping envisionName="dinterface" nwName="dinterface" flags="None" envisionDisplayName="DestinationInterface"/>
 
<mapping envisionName="dmacaddr" nwName="eth.dst" flags="None" format="MAC" envisionDisplayName="DestMacAddress|DestinationMacAddress"/>
 
<mapping envisionName="dmask" nwName="dmask" flags="None"/>
 
<mapping envisionName="dn" nwName="dn" flags="None"/> <mapping envisionName="dst_dn" nwName="dn.dst" flags="None"/> <mapping envisionName="fqdn" nwName="fqdn" flags="None" envisionDisplayName="FQDN"/> <mapping envisionName="src_dn" nwName="dn.src" flags="None"/>
 
<mapping envisionName="dtransport" nwName="dtransport" flags="None"/>
 
<mapping envisionName="event_counter" nwName="event.counter" flags="None" format="Int32"/>
 
<mapping envisionName="filetype" nwName="filetype" flags="None" />
 
<mapping envisionName="gateway" nwName="gateway" flags="None"/>
 
<mapping envisionName="hardware_id" nwName="hardware.id" flags="None"/>
 
<mapping envisionName="icmptype" nwName="icmp.type" flags="None" format="UInt32"/>
 
<mapping envisionName="location_city" nwName="loc.city" flags="None"/>
 
<mapping envisionName="dmacaddr" nwName="eth.dst" flags="None" format="MAC" envisionDisplayName="DestMacAddress|DestinationMacAddress"/> <mapping envisionName="macaddr" nwName="eth.host" flags="None" format="MAC" envisionDisplayName="DeviceMacAddress"/> <mapping envisionName="smacaddr" nwName="eth.src" flags="None" format="MAC" envisionDisplayName="SourceMacAddress" nullTokens="Unknown"/>
 
<mapping envisionName="packets" nwName="packets" flags="None" format="UInt32"/>
 
<mapping envisionName="param_endtime" nwName="param_endtime" flags="None"/>
 
<mapping envisionName="param_event_time" nwName="param_event_time" flags="None"/>
 
<mapping envisionName="param_starttime" nwName="param_starttime" flags="None"/>
 
<mapping envisionName="privilege" nwName="privilege" flags="None" envisionDisplayName="Privilege|Privileges"/>
 
<mapping envisionName="process_id_src" nwName="process.id.src" flags="None" format="Int32" envisionDisplayName="SourceProcessId" nullTokens="(null)|-"/>
 
<mapping envisionName="process_src" nwName="process.src" flags="None" envisionDisplayName="SourceProcess"/>
 
<mapping envisionName="c_domain" nwName="sdomain" flags="None" envisionDisplayName="C_Domain|ClientDomain"/> <mapping envisionName="sdomain" nwName="sdomain" flags="None"/>
 
<mapping envisionName="sessionid" nwName="log.session.id" flags="None"/> <mapping envisionName="sessionid1" nwName="log.session.id1" flags="None"/>
 
<mapping envisionName="sinterface" nwName="sinterface" flags="None" envisionDisplayName="SourceInterface"/>
 
<mapping envisionName="smacaddr" nwName="eth.src" flags="None" format="MAC" envisionDisplayName="SourceMacAddress" nullTokens="Unknown"/>
 
<mapping envisionName="smask" nwName="smask" flags="None"/>
 
<mapping envisionName="timezone" nwName="timezone" flags="None"/>
 
<mapping envisionName="rule_uid" nwName="rule.uid" flags="None"/> <mapping envisionName="uid" nwName="username" flags="None" envisionDisplayName="UserID|UID|Uid" nullTokens="none|-"/>
 
<mapping envisionName="user_org" nwName="org" flags="None" envisionDisplayName="UserOrg|UserOrginization"/>
 
<!-- END List of keys Not in table-map-custom.xml -->

This text should be added to the /etc/netwitness/ng/envision/etc/table-map-custom.xml
After these meta keys are added, the Log Decoder service will need to be restarted.

For more information about the table-map-custom.xml file, see the Maintain Table Map Files in Hosts and Services Getting Started Guide.

Notes

The contents of the attached findmissing.sh script are shown below.

 #!/bin/bash
 
#Script to add show additional meta keys that could be added to table-map-custom.xml file
 
#David Waugh
 
if [ -f /tmp/custom_keys_cef ]
 
then
 
 rm -rf /tmp/custom_keys_cef
 
fi
 
if [ -d /etc/netwitness/ng/envision/etc/devices/cef/ ]
 
then
 
 grep ExtensionKey /etc/netwitness/ng/envision/etc/devices/cef/* |sort | uniq |cut -d " " -f 3 |cut -d \" -
 
f 2|grep -v \< |sort | uniq > /tmp/custom_keys_cef
 
else
 
 echo "CEF Parser is not installed. You can install this parser from RSA Live if you wish."
 
 fi
 
grep "<mapping " /etc/netwitness/ng/envision/etc/table-map.xml | grep -v None |cut -d \" -f 4 >> /tmp/custo
 
m_keys_cef
 
cat /tmp/custom_keys_cef
 
cat /tmp/custom_keys_cef |sort |uniq >/tmp/custom_keys_cef_sorted
 
mv /tmp/custom_keys_cef_sorted /tmp/custom_keys_cef
 
 rm -rf /tmp/TOADD.txt
 
if [ ! -f /etc/netwitness/ng/envision/etc/table-map-custom.xml ]
 
then
 
  echo "You do not have a table-map-custom.xml already defined"
 
else
 
  echo "table-map-custom.xml file is present"
 
fi
 
echo "<!-- BEGIN List of keys Not in table-map-custom.xml -->" >>/tmp/TOADD.txt
 
for metakey in $(cat /tmp/custom_keys_cef)
 
do
 
  METAKEY=$metakey
 
  if [ -f /etc/netwitness/ng/envision/etc/table-map-custom.xml ]
 
  then
 
    COUNTCUSTOM=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map-custom.xml |wc -l)
 
  else
 
    COUNTCUSTOM=0
 
  fi
 
 COUNTTABLEMAP=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map.xml|wc -l)
 
 COUNTISTRANSIENT=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map.xml| grep -v None |wc -l)
 
  echo $metakey $COUNTCUSTOM $COUNTTABLEMAP
 
 # Transient Keys that need to be added that are already in table-map.xml
 
 if [ $COUNTCUSTOM -eq 0 ] && [ $COUNTTABLEMAP -gt 0 ] && [ $COUNTISTRANSIENT -gt 0 ]
 
 then
 
  echo $(grep $metakey  /etc/netwitness/ng/envision/etc/table-map.xml| grep "<mapping ") >>/tmp/TOADD.txt
 
 fi
 
 # Custom Keys that do not exist in table-map.xml at all and need to be added
 
 # Add in the standard Format
 
 if [ $COUNTCUSTOM -eq 0 ] &&  [ $COUNTTABLEMAP -eq 0 ]
 
 then
 
   echo \<mapping envisionName=\"$metakey\" nwName=\"$metakey\" flags=\"None\"\/\> >>/tmp/TOADD.txt
 
 fi
 
done
 
 sed -i -- 's/Transient/None/g' /tmp/TOADD.txt
 
echo "<!-- END List of keys Not in table-map-custom.xml -->" >>/tmp/TOADD.txt
 
echo "Additional Meta keys for table-map-custom.xml can be found in /tmp/TOADD.txt"
 
echo "Paste the contents of this file between the <mappings> </mappings> tags"
 
echo "into the file /etc/netwitness/ng/envision/etc/table-map-custom.xml"

Product Details

RSA Product Set: NetWitness Platform
RSA Product/Service Type: Log Decoder
RSA Version/Condition: 11.x
Platform: CentOS

Approval Reviewer Queue

Technical approval queue