1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
| unit Unit1;
interface
uses
Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
Dialogs, IdBaseComponent, IdComponent, IdTCPConnection, IdTCPClient,
IdHTTP, StdCtrls, URLMon, ComObj, IdCookieManager, IniFiles, Vcl.ExtCtrls, ParseLib;
type
TForm1 = class(TForm)
Memo1: TMemo;
Button1: TButton;
HTTP1: TIdHTTP;
Button3: TButton;
sd: TSaveDialog;
CM: TIdCookieManager;
Timer1: TTimer;
Label1: TLabel;
Edit1: TEdit;
Label2: TLabel;
Edit2: TEdit;
Label3: TLabel;
CheckBox2: TCheckBox;
procedure Button1Click(Sender: TObject);
procedure FormCreate(Sender: TObject);
procedure Button3Click(Sender: TObject);
procedure Timer1Timer(Sender: TObject);
procedure Delay(p: integer);
procedure FormClose(Sender: TObject; var Action: TCloseAction);
private
{ Private declarations }
public
{ Public declarations }
end;
type vac1 = record
price, firma, name, phone, email, link: string;
num: integer;
end;
var
Form1: TForm1;
Excel: variant;
is_run, is_excel, is_stop: boolean;
kol: integer;
ini: TIniFile;
login, pass: string;
is_pause: boolean;
vac: vac1;
captcha: string;
old_ip, new_ip, proxy_server: string;
proxy_port: integer;
pause, pause2: integer;
implementation
{$R *.dfm}
procedure TForm1.Button1Click(Sender: TObject);
var
xx, x1, art, s1, s2, url, url2, page, pages, s, ss, img, img0, im, color, price: string;
i, k: integer;
fl, fl2: boolean;
kart: integer;
j: Integer;
rez, otpr: TStringList;
x: Integer;
a, a0: TextFile;
c: char;
dir: string;
limit: integer;
max_items: integer;
numvac: integer;
fn: string;
jsonToSend:TStringStream;
url0: string;
num_page, max_page: integer;
part: string;
begin
if is_run then Exit;
is_stop:=false;
numvac:=0;
url:=AnsiLowerCase(trim(Edit1.Text));
Edit2.Text:=trim(Edit2.Text);
if Edit2.Text<>'' then
begin
try
limit:=StrToInt(Edit2.Text);
except
limit:=-1;
Edit2.Text:='';
end;
end else limit:=-1;
ini.WriteString('НАСТРОЙКИ', 'лимит', Edit2.Text);
if (url='') then
begin
ShowMessage('Отсутствует ссылка!');
Exit;
end;
if (pos('sakh.com', url)=0) then
begin
ShowMessage('Ссылка некорректна!');
Exit;
end;
if pos('/market', url)>0 then
begin
part:='market';
s1:='<a name="'
end
else
if pos('/sales', url)>0 then
begin
part:='sales';
s1:='<a href="http://auto.sakh.com/sales/cars/?id=';
end
else
begin
ShowMessage('Ссылка некорректна!');
Exit;
end;
url0:=url;
is_run:=true;
http1.Request.UserAgent := 'Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.7.62 Version/11.01';
http1.ProxyParams.Clear;
Application.ProcessMessages;
Memo1.Lines.Add('Начинаем парсинг..');
Application.ProcessMessages;
max_items:=limit;
rez:=TStringList.Create;
fl:=true;
fn:=GetCurrentDir+'\rez'+DateToStr(Date)+'.'+ReplaceStr(TimeToStr(Time), ':', '-')+'.txt';
rez.Clear;
otpr:=TStringList.Create;
otpr.Clear;
if CheckBox2.Checked then
if FileExists('temp.txt') then
otpr.LoadFromFile('temp.txt');
max_items:=-1;
if limit=-1 then max_page:=100000 else max_page:=limit;
num_page:=1;
while (fl) and (not is_stop) and (num_page<=max_page) do
begin
Memo1.Lines.Add('Берем страницу: '+IntToStr(num_page));
Application.ProcessMessages;
url:=ReplaceStr(url, '&', '&');
if (pause<>0)or(pause2<>0) then
Delay(pause+random(pause2));
Application.ProcessMessages;
page:='';
try
page:=http1.Get(url);
except
begin
Application.ProcessMessages;
http1.Disconnect;
Application.ProcessMessages;
sleep(1000);
Application.ProcessMessages;
try
page:=http1.Get(url);
except
begin
Memo1.Lines.Add('Ошибка обращения к '+url);
is_stop:=true;
end;
end;
end;
end;
Application.ProcessMessages;
http1.Request.Referer:=url;
while (ParseAfter(page, s1)) and (not is_stop) do
begin
// if numvac>100 then break;
Application.ProcessMessages;
if is_stop then break;
ss:=page;
ParseBefore(ss, '"');
ParseBefore(ss, '&');
ss:=trim(ss);
Application.ProcessMessages;
if ss<>'' then
if otpr.IndexOf('http://auto.sakh.com/'+part+'/'+ss)=-1 then
begin
url:='http://auto.sakh.com/'+part+'/'+ss;
vac.num:=vac.num+1;
vac.link:=url;
ParseBefore(vac.link, '?');
vac.email:='';
Application.ProcessMessages;
if is_stop then break;
x1:=url;
ParseBefore(x1, '?');
ParseLastAfter(x1, '/', true);
if x1<>'' then
begin
if (pause<>0)or(pause2<>0) then
Delay(pause+random(pause2));
Application.ProcessMessages;
http1.Request.Referer:=url;
Application.ProcessMessages;
try
x1:=http1.Get('http://auto.sakh.com/ajax.php?action=getContacts&offer_id='+x1+'&part='+part);
except
Memo1.Lines.Add('Ошибка получение e-mail.');
x1:='';
end;
http1.Request.Referer:=url0;
Application.ProcessMessages;
vac.email:='';
while ParseAfter(x1, 'E-mail" class=likelink>') do
begin
ss:=x1;
if ParseBefore(ss, '<', true) then
if pos('@', ss)>0 then
if rez.IndexOf(ss)=-1 then
begin
rez.Add(ss);
rez.SaveToFile(fn);
numvac:=numvac+1;
end;
end;
end;
if x1<>'' then
begin
otpr.Add(vac.link);
otpr.SaveToFile('temp.txt');
end;
end;
end;
num_page:=num_page+1;
if not is_stop then
begin
if (vac.num<max_items)or(max_items=-1) then
begin
url:='';
pages:=page;
if ParseBefore(pages, '">следующая »</a>') then
begin
ParseLastAfter(pages, '<a href="');
if pages<>'' then
url:=pages else url:='';
url:=ReplaceStr(url, '&', '&');
end;
if url='' then fl:=false;
end else fl:=false;
end;
end;
Memo1.Lines.Add('Взято '+IntToStr(numvac)+' е-мейлов.');
Memo1.Lines.Add('Сохранено!');
if not is_stop then
begin
Memo1.Lines.Add('Завершено.');
end else
begin
Memo1.Lines.Add('Остановлено.');
end;
is_run:=false;
otpr.Free;
end;
procedure TForm1.Button3Click(Sender: TObject);
begin
is_stop:=true;
end;
procedure TForm1.FormCreate(Sender: TObject);
begin
randomize;
is_run:=false;
is_stop:=false;
ini:=TIniFile.Create(GetCurrentDir+'\parser.ini');
Edit1.Text:=ini.ReadString('НАСТРОЙКИ', 'URL', '');
Edit2.Text:=ini.ReadString('НАСТРОЙКИ', 'лимит', '');
pause:=ini.ReadInteger('НАСТРОЙКИ', 'пауза', 0);
pause2:=ini.ReadInteger('НАСТРОЙКИ', 'случайная пауза', 0);
end;
procedure TForm1.Delay(p: integer);
begin
Timer1.Interval:=p;
is_pause:=true;
Timer1.Enabled:=true;
while (is_pause)and(not is_stop) do
Application.ProcessMessages;
end;
procedure TForm1.Timer1Timer(Sender: TObject);
begin
Timer1.Enabled:=false;
is_pause:=false;
end;
procedure TForm1.FormClose(Sender: TObject; var Action: TCloseAction);
begin
ini.Free;
end;
end. |
(
Сообщение было отмечено Favorskij как решение
Сообщение от qwertehok
