WARN YarnScheduler: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered

29 Views Asked by At

I run spark-submit --master spark then success but run spark-submit --master yarn then error

Any spark jobs that I run will fail with the following error message

24/01/10 14:46:24 WARN YarnScheduler: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources

24/01/10 14:46:39 WARN YarnScheduler: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources

This file yarn-site:

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>
 <property>
         <name>yarn.nodemanager.aux-services</name>
         <value>mapreduce_shuffle</value>
     </property>
     <property>
         <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
         <value>org.apache.hadoop.mapred.ShuffleHandler</value>
     </property>
     <property>
       <name>yarn.resourcemanager.hostname</name>
       <value>192.168.71.190</value>
    </property> 
   <property>
  <name>yarn.acl.enable</name>
  <value>0</value>
</property>
<property>
  <name>yarn.nodemanager.env-whitelist</name>   
  <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PERPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
</property>
<property>
    <name>yarn.nodemanager.resource.cpu-vcores</name>
    <value>8</value>
</property>
<property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>16384</value>
</property>
<property>
    <name>yarn.scheduler.maximum-allocation-mb</name>
    <value>16384</value>
</property>
<property>
    <name>yarn.scheduler.minimum-allocation-mb</name>
    <value>512</value>
</property>
<property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
</property>

<property>
                <name>yarn.resourcemanager.address</name>
                <value>192.168.71.190:8032</value>
        </property>
<!-- Site specific YARN configuration properties -->

</configuration>

This file spark-env:

export HADOOP_CONF_DIR=/app/hadoop-3.3.6/etc/hadoop

export YARN_CONF_DIR=/app/hadoop-3.3.6/etc/hadoop

export SPARK_MASTER_HOST=192.168.71.190

export LD_LIBRARY_PATH="/app/hadoop-3.3.6/lib/native:${LD_LIBRARY_PATH}"

export JAVA_HOME=/app/jdk-11.0.20/

This file spark-defaults:

spark.blockManager.port 30000

spark.broadcast.port 30001

spark.driver.port 30002

\#spark.dynamicAllocation.portMin=30003

\#spark.dynamicAllocation.portMax=30003

spark.fileserver.port 30004

spark.replClassServer.port 30005

spark.shuffle.service.port 30003

spark.master.port 7077

spark.master    yarn

spark.driver.memory 4g

spark.yarn.am.memory 4g

spark.executor.memory 4g

Please help me, I really need it

0

There are 0 best solutions below