Hadoop 上使用C 語言編程

今天嘗試用C語言在Hadoop上編寫統計單詞的程序，具體過程如下：

一、編寫map和reduce程序

mapper.c

1. #include <stdio.h>
2. #include <stdlib.h>
3. #include <string.h>
5. #define BUF_SIZE 2048
6. #define DELIM '\n'
8. int main(int argc, char * argv[])
9. {
10. char buffer[BUF_SIZE];
11. while(fgets(buffer,BUF_SIZE-1,stdin))
12. {
13. int len = strlen(buffer);
14. if(buffer[len-1] == DELIM) // 將換行符去掉
15. buffer[len-1] = 0;
17. char *query = NULL;
18. query = strtok(buffer, " ");
19. while(query)
20. {
21. printf("%s\t1\n",query);
22. query = strtok(NULL," ");
23. }
24. }
25. return 0;
26. }

1. #include <stdio.h>
2. #include <stdlib.h>
3. #include <string.h>
5. #define BUFFER_SIZE 1024
6. #define DELIM "\t"
8. int main(int argc, char * argv[])
9. {
10. char str_last_key[BUFFER_SIZE];
11. char str_line[BUFFER_SIZE];
12. int count = 0;
14. *str_last_key = '\0';
16. while( fgets(str_line,BUFFER_SIZE-1,stdin) )
17. {
18. char * str_cur_key = NULL;
19. char * str_cur_num = NULL;
21. str_cur_key = strtok(str_line,DELIM);
22. str_cur_num = strtok(NULL,DELIM);
24. if(str_last_key[0] =='\0')
25. {
26. strcpy(str_last_key,str_cur_key);
27. }
28. if(strcmp(str_cur_key, str_last_key))// 前後不相等，輸出
29. {
30. printf("%s\t%d\n",str_last_key,count);
31. count = atoi(str_cur_num);
32. }else{// 相等，則加當前的key的value
33. count += atoi(str_cur_num);
34. }
35. strcpy(str_last_key,str_cur_key);
36. }
37. printf("%s\t%d\n",str_last_key,count);
38. return 0;
39. }

gcc mapper.c -o mapper

gcc reducer.c -o reducer

三、運行

（一）啟動hadoop後將待統計單詞的輸入文件放到 input文件夾中：bin/hadoop fs -put 待統計文件 input

（二）使用contrib/streaming/下的jar工具調用上面的mapper\reducer:

bin/hadoop jar /home/huangkq/Desktop/hadoop/contrib/streaming/hadoop-streaming-0.20.203.0.jar -mapper /home/huangkq/Desktop/hadoop2/mapper -reducer /home/huangkq/Desktop/hadoop2/reducer -input input -output c_output -jobconf mapred.reduce.tasks=2

說明：hadoop-streaming-0.20.203.0.jar是一個管道工具

（三）查看結果：bin/hadoop fs -cat c_output/*

更多Hadoop相關信息見Hadoop 專題頁面 http://www.linuxidc.com/topicnews.aspx?tid=13