HDP 2.4.2: KerberosクラスターでSparkのExample HBaseTestを使う

# 必要なし：ln -sf /etc/hbase/conf/hbase-site.xml /etc/spark/conf/hbase-site.xml
# うまくいかない：export SPARK_CLASSPATH="`hbase classpath`"

export SPARK_CLASSPATH="$SPARK_CLASSPATH:/usr/hdp/current/hbase-client/lib/hbase-client.jar:/usr/hdp/current/hbase-client/lib/hbase-common.jar:/usr/hdp/current/hbase-client/lib/hbase-server.jar:/usr/hdp/current/hbase-client/lib/guava-12.0.1.jar:/usr/hdp/current/hbase-client/lib/hbase-protocol.jar:/usr/hdp/current/hbase-client/lib/htrace-core-3.1.0-incubating.jar:/etc/hbase/conf/hbase-site.xml"

# ambarismoketestを先に作らないとだめ？

spark-submit --verbose --class org.apache.spark.examples.HBaseTest --master yarn-cluster --jars /usr/hdp/current/hbase-client/lib/hbase-client.jar,/usr/hdp/current/hbase-client/lib/hbase-common.jar,/usr/hdp/current/hbase-client/lib/hbase-server.jar,/usr/hdp/current/hbase-client/lib/guava-12.0.1.jar,/usr/hdp/current/hbase-client/lib/hbase-protocol.jar,/usr/hdp/current/hbase-client/lib/htrace-core-3.1.0-incubating.jar --files /etc/hbase/conf/hbase-site.xml --keytab /etc/security/keytabs/smokeuser.headless.keytab --principal ambari-qa-bne_c1@HO-UBU02 /usr/hdp/current/spark-client/lib/spark-examples-*.jar ambarismoketest --conf spark.driver.extraJavaOptions = "-Dsun.security.krb5.debug=true -Dsun.security.spnego.debug=true"

# KeytabとPrincipalなしでも動く

kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa-bne_c1

spark-submit --verbose --class org.apache.spark.examples.HBaseTest --master yarn-cluster --jars /usr/hdp/current/hbase-client/lib/hbase-client.jar,/usr/hdp/current/hbase-client/lib/hbase-common.jar,/usr/hdp/current/hbase-client/lib/hbase-server.jar,/usr/hdp/current/hbase-client/lib/guava-12.0.1.jar,/usr/hdp/current/hbase-client/lib/hbase-protocol.jar,/usr/hdp/current/hbase-client/lib/htrace-core-3.1.0-incubating.jar --files /etc/hbase/conf/hbase-site.xml /usr/hdp/current/spark-client/lib/spark-examples-*.jar ambarismoketest

備考：Yarn Application LogにDEBUGを出力するには、/etc/spark/conf/log4j.propertiesの変更が必要
log4j.rootCategory=DEBUG, console (https://issues.apache.org/jira/browse/SPARK-12279)