sent 44 bytes received 12 bytes 37.33 bytes/sec total size is 740 speedup is 13.21 ==================== hadoop103 ==================== sending incremental file list xsync
sent 831 bytes received 35 bytes 577.33 bytes/sec total size is 740 speedup is 0.85 ==================== hadoop104 ==================== sending incremental file list xsync
sent 831 bytes received 35 bytes 577.33 bytes/sec total size is 740 speedup is 0.85
#!/bin/bash #1. 判断参数个数 if [ $# -lt 1 ] then echo Not Enough Arguement! exit; fi #2. 遍历集群所有机器 for host in hadoop102 hadoop103 hadoop104 do echo ==================== $host ==================== #3. 遍历所有目录，挨个发送
for file in $@ do #4. 判断文件是否存在 if [ -e $file ] then #5. 获取父目录 pdir=$(cd -P $(dirname $file); pwd)
#6. 获取当前文件的名称 fname=$(basename $file) ssh $host "mkdir -p $pdir" rsync -av $pdir/$fname $host:$pdir else echo $file does not exists! fi done done
# 执行日志 2022-04-19 10:43:28,907 INFO client.RMProxy: Connecting to ResourceManager at hadoop103/192.168.3.103:8032 2022-04-19 10:43:29,574 INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for path: /tmp/hadoop-yarn/staging/root/.staging/job_1650334953814_0001 2022-04-19 10:43:29,713 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false 2022-04-19 10:43:29,973 INFO input.FileInputFormat: Total input files to process : 1 2022-04-19 10:43:30,023 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false 2022-04-19 10:43:30,104 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false 2022-04-19 10:43:30,155 INFO mapreduce.JobSubmitter: number of splits:1 2022-04-19 10:43:30,326 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false 2022-04-19 10:43:30,420 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1650334953814_0001 2022-04-19 10:43:30,420 INFO mapreduce.JobSubmitter: Executing with tokens: [] 2022-04-19 10:43:30,700 INFO conf.Configuration: resource-types.xml not found 2022-04-19 10:43:30,701 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'. 2022-04-19 10:43:31,208 INFO impl.YarnClientImpl: Submitted application application_1650334953814_0001 2022-04-19 10:43:31,279 INFO mapreduce.Job: The url to track the job: http://hadoop103:8088/proxy/application_1650334953814_0001/ 2022-04-19 10:43:31,280 INFO mapreduce.Job: Running job: job_1650334953814_0001 2022-04-19 10:43:41,697 INFO mapreduce.Job: Job job_1650334953814_0001 running in uber mode : false 2022-04-19 10:43:41,699 INFO mapreduce.Job: map 0% reduce 0% 2022-04-19 10:43:50,951 INFO mapreduce.Job: map 100% reduce 0% 2022-04-19 10:44:00,224 INFO mapreduce.Job: map 100% reduce 100% 2022-04-19 10:44:01,259 INFO mapreduce.Job: Job job_1650334953814_0001 completed successfully 2022-04-19 10:44:01,402 INFO mapreduce.Job: Counters: 53 File System Counters FILE: Number of bytes read=42 FILE: Number of bytes written=434299 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=139 HDFS: Number of bytes written=24 HDFS: Number of read operations=8 HDFS: Number of large read operations=0 HDFS: Number of write operations=2 Job Counters Launched map tasks=1 Launched reduce tasks=1 Data-local map tasks=1 Total time spent by all maps in occupied slots (ms)=25120 Total time spent by all reduces in occupied slots (ms)=22660 Total time spent by all map tasks (ms)=6280 Total time spent by all reduce tasks (ms)=5665 Total vcore-milliseconds taken by all map tasks=6280 Total vcore-milliseconds taken by all reduce tasks=5665 Total megabyte-milliseconds taken by all map tasks=25722880 Total megabyte-milliseconds taken by all reduce tasks=23203840 Map-Reduce Framework Map input records=3 Map output records=6 Map output bytes=60 Map output materialized bytes=42 Input split bytes=103 Combine input records=6 Combine output records=3 Reduce input groups=3 Reduce shuffle bytes=42 Reduce input records=3 Reduce output records=3 Spilled Records=6 Shuffled Maps =1 Failed Shuffles=0 Merged Map outputs=1 GC time elapsed (ms)=306 CPU time spent (ms)=2120 Physical memory (bytes) snapshot=445448192 Virtual memory (bytes) snapshot=10477551616 Total committed heap usage (bytes)=394264576 Peak Map Physical memory (bytes)=275689472 Peak Map Virtual memory (bytes)=5234483200 Peak Reduce Physical memory (bytes)=169758720 Peak Reduce Virtual memory (bytes)=5243068416 Shuffle Errors BAD_ID=0 CONNECTION=0 IO_ERROR=0 WRONG_LENGTH=0 WRONG_MAP=0 WRONG_REDUCE=0 File Input Format Counters Bytes Read=36 File Output Format Counters Bytes Written=24
