shell操作文件的幾條命令：刪除最後一列、刪除第一行、diff等
刪除文件第一行： sed '1d' filename
刪除文件最後一列： awk '{print $NF}' filename
比較文件的兩種方法：
1）comm -3 --nocheck-order file1 file2
2) grep -v -f file1 file2 :輸出file2中有file1中沒有的行
當然還有diff file1 file2
貼一段昨天寫的shell腳本~
#!/bin/bash
date_time=`date +'%H_%M_%S'`
yesterday=`date -d"-1 day" +'%Y_%m_%d'`
today=`date +'%Y_%m_%d'`
date_day_time=`date +'%Y_%m_%d_%H_%M_%S'`
mkdir /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/same_similiar_log/$today
# begin to get input files which haven't been deal with
today_input=/home/crawler/petabyte/crawllog/news_data/$today
yesterday_input=/home/crawler/petabyte/crawllog/news_data/$yesterday
/opt/hadoop/program/bin/hadoop fs -ls $yesterday_input/ > /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/all_get
/opt/hadoop/program/bin/hadoop fs -ls $today_input/ >> /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/all_get
sed '1d' /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/all_get > /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/all_get_without_first_line
awk '{print $NF}' /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/all_get_without_first_line > /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/all_input
#comm -3 --nocheck-order /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/all_input /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/input_done > /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/today_diff
grep -v -f /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/input_done /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/all_input > /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/today_diff
awk '{print $NF}' /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/today_diff > /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/today_new_input
mv /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/all_input /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/input_done
# begin to compute same_similary_news
inputfile1=""
while read line
do
inputfile1=$inputfile1,${line}
done < /home/spamdetect/changxiaojia/workspace/finance/same_similar_news_mining/mid_files/input_done
echo $inputfile1
