-
Notifications
You must be signed in to change notification settings - Fork 2
/
ftp2Local2Hdfs.kjb
206 lines (205 loc) · 9.89 KB
/
ftp2Local2Hdfs.kjb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
<?xml version="1.0" encoding="UTF-8"?>
<job>
<name>ftp2Local2Hdfs</name>
<description/>
<extended_description/>
<job_version/>
<job_status>0</job_status>
<directory>/</directory>
<created_user>-</created_user>
<created_date>2015/02/09 16:53:37.064</created_date>
<modified_user>-</modified_user>
<modified_date>2016/10/07 12:13:41.926</modified_date>
<parameters>
<parameter>
<name>ftpHdfsInputPath</name>
<default_value>hdfs://hadoop01:9000/compareFtp/</default_value>
<description/>
</parameter>
<parameter>
<name>inputfiledir</name>
<default_value>/home/test/import</default_value>
<description/>
</parameter>
<parameter>
<name>inputfilename</name>
<default_value>import.txt</default_value>
<description/>
</parameter>
<parameter>
<name>inputftpIp</name>
<default_value>192.168.0.31</default_value>
<description/>
</parameter>
<parameter>
<name>inputftppasswd</name>
<default_value>redhat</default_value>
<description/>
</parameter>
<parameter>
<name>inputftpport</name>
<default_value>21</default_value>
<description/>
</parameter>
<parameter>
<name>inputftpusername</name>
<default_value>root</default_value>
<description/>
</parameter>
<parameter>
<name>targetdirectory</name>
<default_value>/home/jimmy</default_value>
<description/>
</parameter>
</parameters>
<slaveservers>
</slaveservers>
<job-log-table><connection/>
<schema/>
<table/>
<size_limit_lines/>
<interval/>
<timeout_days/>
<field><id>ID_JOB</id><enabled>Y</enabled><name>ID_JOB</name></field><field><id>CHANNEL_ID</id><enabled>Y</enabled><name>CHANNEL_ID</name></field><field><id>JOBNAME</id><enabled>Y</enabled><name>JOBNAME</name></field><field><id>STATUS</id><enabled>Y</enabled><name>STATUS</name></field><field><id>LINES_READ</id><enabled>Y</enabled><name>LINES_READ</name></field><field><id>LINES_WRITTEN</id><enabled>Y</enabled><name>LINES_WRITTEN</name></field><field><id>LINES_UPDATED</id><enabled>Y</enabled><name>LINES_UPDATED</name></field><field><id>LINES_INPUT</id><enabled>Y</enabled><name>LINES_INPUT</name></field><field><id>LINES_OUTPUT</id><enabled>Y</enabled><name>LINES_OUTPUT</name></field><field><id>LINES_REJECTED</id><enabled>Y</enabled><name>LINES_REJECTED</name></field><field><id>ERRORS</id><enabled>Y</enabled><name>ERRORS</name></field><field><id>STARTDATE</id><enabled>Y</enabled><name>STARTDATE</name></field><field><id>ENDDATE</id><enabled>Y</enabled><name>ENDDATE</name></field><field><id>LOGDATE</id><enabled>Y</enabled><name>LOGDATE</name></field><field><id>DEPDATE</id><enabled>Y</enabled><name>DEPDATE</name></field><field><id>REPLAYDATE</id><enabled>Y</enabled><name>REPLAYDATE</name></field><field><id>LOG_FIELD</id><enabled>Y</enabled><name>LOG_FIELD</name></field><field><id>EXECUTING_SERVER</id><enabled>N</enabled><name>EXECUTING_SERVER</name></field><field><id>EXECUTING_USER</id><enabled>N</enabled><name>EXECUTING_USER</name></field><field><id>START_JOB_ENTRY</id><enabled>N</enabled><name>START_JOB_ENTRY</name></field><field><id>CLIENT</id><enabled>N</enabled><name>CLIENT</name></field></job-log-table>
<jobentry-log-table><connection/>
<schema/>
<table/>
<timeout_days/>
<field><id>ID_BATCH</id><enabled>Y</enabled><name>ID_BATCH</name></field><field><id>CHANNEL_ID</id><enabled>Y</enabled><name>CHANNEL_ID</name></field><field><id>LOG_DATE</id><enabled>Y</enabled><name>LOG_DATE</name></field><field><id>JOBNAME</id><enabled>Y</enabled><name>TRANSNAME</name></field><field><id>JOBENTRYNAME</id><enabled>Y</enabled><name>STEPNAME</name></field><field><id>LINES_READ</id><enabled>Y</enabled><name>LINES_READ</name></field><field><id>LINES_WRITTEN</id><enabled>Y</enabled><name>LINES_WRITTEN</name></field><field><id>LINES_UPDATED</id><enabled>Y</enabled><name>LINES_UPDATED</name></field><field><id>LINES_INPUT</id><enabled>Y</enabled><name>LINES_INPUT</name></field><field><id>LINES_OUTPUT</id><enabled>Y</enabled><name>LINES_OUTPUT</name></field><field><id>LINES_REJECTED</id><enabled>Y</enabled><name>LINES_REJECTED</name></field><field><id>ERRORS</id><enabled>Y</enabled><name>ERRORS</name></field><field><id>RESULT</id><enabled>Y</enabled><name>RESULT</name></field><field><id>NR_RESULT_ROWS</id><enabled>Y</enabled><name>NR_RESULT_ROWS</name></field><field><id>NR_RESULT_FILES</id><enabled>Y</enabled><name>NR_RESULT_FILES</name></field><field><id>LOG_FIELD</id><enabled>N</enabled><name>LOG_FIELD</name></field><field><id>COPY_NR</id><enabled>N</enabled><name>COPY_NR</name></field></jobentry-log-table>
<channel-log-table><connection/>
<schema/>
<table/>
<timeout_days/>
<field><id>ID_BATCH</id><enabled>Y</enabled><name>ID_BATCH</name></field><field><id>CHANNEL_ID</id><enabled>Y</enabled><name>CHANNEL_ID</name></field><field><id>LOG_DATE</id><enabled>Y</enabled><name>LOG_DATE</name></field><field><id>LOGGING_OBJECT_TYPE</id><enabled>Y</enabled><name>LOGGING_OBJECT_TYPE</name></field><field><id>OBJECT_NAME</id><enabled>Y</enabled><name>OBJECT_NAME</name></field><field><id>OBJECT_COPY</id><enabled>Y</enabled><name>OBJECT_COPY</name></field><field><id>REPOSITORY_DIRECTORY</id><enabled>Y</enabled><name>REPOSITORY_DIRECTORY</name></field><field><id>FILENAME</id><enabled>Y</enabled><name>FILENAME</name></field><field><id>OBJECT_ID</id><enabled>Y</enabled><name>OBJECT_ID</name></field><field><id>OBJECT_REVISION</id><enabled>Y</enabled><name>OBJECT_REVISION</name></field><field><id>PARENT_CHANNEL_ID</id><enabled>Y</enabled><name>PARENT_CHANNEL_ID</name></field><field><id>ROOT_CHANNEL_ID</id><enabled>Y</enabled><name>ROOT_CHANNEL_ID</name></field></channel-log-table>
<pass_batchid>N</pass_batchid>
<shared_objects_file/>
<entries>
<entry>
<name>START</name>
<description/>
<type>SPECIAL</type>
<start>Y</start>
<dummy>N</dummy>
<repeat>N</repeat>
<schedulerType>0</schedulerType>
<intervalSeconds>0</intervalSeconds>
<intervalMinutes>60</intervalMinutes>
<hour>12</hour>
<minutes>0</minutes>
<weekDay>1</weekDay>
<DayOfMonth>1</DayOfMonth>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>123</xloc>
<yloc>88</yloc>
</entry>
<entry>
<name>成功</name>
<description/>
<type>SUCCESS</type>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>708</xloc>
<yloc>88</yloc>
</entry>
<entry>
<name>从FTP 下载文件</name>
<description/>
<type>FTP</type>
<port>${inputftpport}</port>
<servername>${inputftpIp}</servername>
<username>${inputftpusername}</username>
<password>${inputftppasswd}</password>
<ftpdirectory>${inputfiledir}</ftpdirectory>
<targetdirectory>${targetdirectory}</targetdirectory>
<wildcard>${inputfilename}.*</wildcard>
<binary>N</binary>
<timeout>0</timeout>
<remove>N</remove>
<only_new>N</only_new>
<active>N</active>
<control_encoding>ISO-8859-1</control_encoding>
<movefiles>N</movefiles>
<movetodirectory/>
<adddate>N</adddate>
<addtime>N</addtime>
<SpecifyFormat>N</SpecifyFormat>
<date_time_format/>
<AddDateBeforeExtension>N</AddDateBeforeExtension>
<isaddresult>Y</isaddresult>
<createmovefolder>N</createmovefolder>
<proxy_host/>
<proxy_port/>
<proxy_username/>
<proxy_password>Encrypted </proxy_password>
<socksproxy_host/>
<socksproxy_port>1080</socksproxy_port>
<socksproxy_username/>
<socksproxy_password>Encrypted </socksproxy_password>
<ifFileExists>ifFileExistsSkip</ifFileExists>
<nr_limit>10</nr_limit>
<success_condition>success_if_no_errors</success_condition>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>288</xloc>
<yloc>88</yloc>
</entry>
<entry>
<name>Shell</name>
<description/>
<type>SHELL</type>
<filename/>
<work_directory/>
<arg_from_previous>N</arg_from_previous>
<exec_per_row>N</exec_per_row>
<set_logfile>N</set_logfile>
<logfile/>
<set_append_logfile>N</set_append_logfile>
<logext/>
<add_date>N</add_date>
<add_time>N</add_time>
<insertScript>Y</insertScript>
<script>hadoop fs -rm -r ${ftpHdfsInputPath}
hadoop fs -mkdir ${ftpHdfsInputPath}
hadoop fs -put ${targetdirectory}/${inputfilename} ${ftpHdfsInputPath}</script>
<loglevel>Basic</loglevel>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>498</xloc>
<yloc>88</yloc>
</entry>
</entries>
<hops>
<hop>
<from>START</from>
<to>从FTP 下载文件</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>从FTP 下载文件</from>
<to>Shell</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>Shell</from>
<to>成功</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
</hops>
<notepads>
</notepads>
</job>