lib/itamae/plugin/recipe/spark/install.rb in itamae-plugin-recipe-spark-0.1.4 vs lib/itamae/plugin/recipe/spark/install.rb in itamae-plugin-recipe-spark-0.1.5
- old
+ new
@@ -1,15 +1,22 @@
version = ENV['SPARK_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_VERSION
hadoop_version = ENV['HADOOP_VERSION'] || Itamae::Plugin::Recipe::Hadoop::HADOOP_VERSION
+hadoop_type = if Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
+ '3.2'
+ elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('2.7')
+ '2.7'
+ else
+ raise "Hadoop version #{hadoop_version} is not supported."
+ end
execute "download spark-#{version}" do
cwd '/tmp'
command <<-EOF
- rm -f spark-#{version}-bin-hadoop2.7.tgz
- wget https://archive.apache.org/dist/spark/spark-#{version}/spark-#{version}-bin-hadoop2.7.tgz
+ rm -f spark-#{version}-bin-hadoop#{hadoop_type}.tgz
+ wget https://archive.apache.org/dist/spark/spark-#{version}/spark-#{version}-bin-hadoop#{hadoop_type}.tgz
EOF
- not_if "test -e /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_sha256.txt")).strip} | sha256sum -c"
+ not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
end
directory '/opt/spark' do
user 'root'
owner 'root'
@@ -18,40 +25,40 @@
end
execute "install spark-#{version}" do
cwd '/tmp'
command <<-EOF
- rm -Rf spark-#{version}-bin-hadoop2.7/
- tar zxf spark-#{version}-bin-hadoop2.7.tgz
- sudo rm -Rf /opt/spark/spark-#{version}-bin-hadoop2.7
- sudo mv spark-#{version}-bin-hadoop2.7 /opt/spark/
- sudo touch /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED
+ rm -Rf spark-#{version}-bin-hadoop#{hadoop_type}/
+ tar zxf spark-#{version}-bin-hadoop#{hadoop_type}.tgz
+ sudo rm -Rf /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}
+ sudo mv spark-#{version}-bin-hadoop#{hadoop_type} /opt/spark/
+ sudo touch /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED
EOF
- not_if "test -e /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED"
+ not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED"
end
execute 'install hadoop aws jars' do
cwd '/opt/hadoop/current'
command <<-EOF
cp -f share/hadoop/tools/lib/aws-java-sdk-*.jar \
- /opt/spark/spark-#{version}-bin-hadoop2.7/jars/
+ /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
cp -f share/hadoop/tools/lib/hadoop-aws-#{hadoop_version}.jar \
- /opt/spark/spark-#{version}-bin-hadoop2.7/jars/
+ /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
EOF
- not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop2.7/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
+ not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
end
-template "/opt/spark/spark-#{version}-bin-hadoop2.7/conf/spark-defaults.conf"
+template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/spark-defaults.conf"
if ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY']
- template "/opt/spark/spark-#{version}-bin-hadoop2.7/conf/hdfs-site.xml" do
+ template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/hdfs-site.xml" do
variables aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
end
end
link '/opt/spark/current' do
- to "/opt/spark/spark-#{version}-bin-hadoop2.7"
+ to "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
user 'root'
force true
end