Анализ информации из социальных сетей

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Oct 29 09:11:31 2018
 
@author: irip
"""
import requests
import json
from time import sleep
import csv
 
def write_json(data):
    with open('posts.json', 'w') as file:
        json.dump(data, file, indent = 2, ensure_ascii = False)
   
 
def to_json(post_dict):
    try:
        data = json.load(open('posts_data.json'))
    except:
        data = []
    
    data.append(post_dict)
    
    with open('posts_data.json', 'w') as file:
        json.dumo(data, file, indent = 2, ensure_ascii = False)
 
 
def write_csv(data):
    with open('posts_data.csv', 'a') as file:
        writer = csv.writer(file)
        
        writer.writerow((
            data['id'], 
            data['date'], 
            data['attachments'],
            data['text']
        ))
   
   
def get_data(post):
    try:
        post_id = post['id']
    except:
        post_id = 'null'
    
    try:
        post_date = post['date']
    except:
        post_date = 'null'
    
    try:
        post_text = post['text']
    except:
        post_text = 'Блог Олеси'
        
    try:
        post_attachments = post['attachments']
    except:
        post_attachments = '* * *'
    
    data = {
        'id': post_id,
        'date': post_date,
        'attachments': post_attachments,
        'text': post_text
    }
    
    return data
 
def main():
    # https://api.vk.com/method/wall.get?user_id=210700286&v=5.52
    
    group_id = '-108964054'
    
    token = ' '
    offset = 0
    date_start = 1456978229
    all_posts = []
    
    while True:
        sleep(1)
        r = requests.get(
            'https://api.vk.com/method/wall.get', 
            params = {
                'owner_id': group_id,
                'count': 100,
                'offset': offset,
                'access_token':token,
                'version':5.58
            }
        )
        posts = r.json()['response']
        all_posts.extend(posts)
        offset += 100
        print(offset)
        
        start_post = posts[-1]['date']
        
        if start_post < date_x:
            break
        
        
    data_posts = []
    
    for post in all_posts:
        post_data = get_data(post)
        print(post_data)
        write_csv(post_data)
        
        
 
if __name__ == '__main__':
    main()

>>> def get_data(post):
    
    return  {
        'id': post.get('post_id','null') ,
        'date': post.get('post_date','null'),
        'attachments':post.get('post_attachments','null'),
        'text':post.get('post_text','null')
        }  if isinstance(post,dict) else {}
 
>>> get_data(1)
{}
>>> get_data(None)
{}
>>> get_data({'post_id':100500})
{'id': 100500, 'date': 'null', 'attachments': 'null', 'text': 'null'}
>>> get_data({'post_id':100500,'post_date':'2018-10-29','всяко_разно':"123"})
{'id': 100500, 'date': '2018-10-29', 'attachments': 'null', 'text': 'null'}

for post in all_posts:
    post_data = get_data(post)
    print(post_data)
    write_csv(post_data)
    for attachment in post_data['post_attachments']:
        att = get_attachment_data(attachment)
        write_csv_attachment(post_data)

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Oct 29 09:11:31 2018
 
@author: irip
"""
import requests
import json
from time import sleep
import csv
from conf import id as source_id
from conf import token as token
 
api_v = 5.58
 
params = {
    'v': api_v,
    'access_token': token,
    'user_id': source_id,
    'count': 200,
    'offset': 0,
    'rev': 1
}
 
def main():
    # https://api.vk.com/method/wall.get?user_id=210700286&v=5.52
    
    offset = 0
    date_x = 1456978229
    all_posts = []
    
    while True:
        sleep(1)
        r = requests.get(
            'https://api.vk.com/method/wall.get', 
            params = {
                'owner_id': source_id,
                'count': 100,
                'offset': offset,
                'access_token':token,
                'version': api_v
            }
        )
        posts = r.json()['response']
        all_posts.extend(posts)
        offset += 100
        print(offset)
        
        start_post = posts[-1]['date']
        
        if start_post < date_x:
            break
        
        
    data_posts = []
    #print(all_posts)
    for post in all_posts:
        print(post)
        post_data = get_data(post)
        # print(post_data)
        save_csv(post_data)
        print('done!')
        #for attachment in post_data['attachments']:
                #att = get_attachment_data(attachment)
                #write_csv_attachment(post_data)        
 
 
 
   
def get_data(post):
    try:
        post_id = post['id']
    except:
        post_id = 'null'
    
    try:
        post_date = post['date']
    except:
        post_date = 'null'
   
    try:
        post_attachments = post['attachments']
    except:
        post_attachments = '* * *'
        
    try:
        post_text = post['text']
    except:
        post_text = '* * *'
        
 
    data = {
        'id': post_id,
        'date': post_date,
        'attachments': post_attachments,
        'text': post_text
    }
    
    return data
 
 
def get_attachment_data(post):
    """
        Парсим аттачи
        Возвращаем массивы прикрепленных урлов att
        wall_id = USER_ID
        message_id = POST_ID
    """
    if post["type"] == "photo":
        if post["photo"]["id"] != 0:
            save_photo('%s_%s_%s' % (
                post["photo"]["owner_id"], 
                post["photo"]["id"], 
                post["photo"]["access_key"]
            ))
            att = Attachment(
                type_id = 2,
                name = post["photo"]["id"],
                link = "img/%s.jpg" % a["photo"]["id"],
                message_id = msg_id,
                wall_id = wall_id
            )
            session.add(att)
    elif post["type"] == "video":
        link = get_video_link('%s_%s_%s' % (
            post["video"]["owner_id"], 
            post["video"]["id"], 
            post["video"]["access_key"]
        ))
        att = Attachment(
            type_id = 3,
            name = post["video"]["title"],
            link = link,
            message_id = msg_id,
            wall_id = wall_id
        )
        session.add(att)
    elif post["type"] == "audio":
        att = Attachment(
            type_id = 1,
            name = post['audio']['artist'] + post['audio']['title'],
            message_id = msg_id,
            wall_id = wall_id
        )
        session.add(att)
    elif post["type"] == "doc":
        save_doc('%s_%s_%s' % (
            post["doc"]["owner_id"], 
            post["doc"]["id"], 
            post["doc"]["access_key"]
        ))
        att = Attachment(
            type_id = 4,
            name = post['doc']['title'],
            message_id = msg_id,
            wall_id = wall_id
        )
        session.add(att)
    elif post["type"] == "link":
        att = Attachment(
            type_id = 5,
            name = post["link"]["url"],
            link = post["link"]["url"],
            message_id = msg_id,
            wall_id = wall_id
        )
        session.add(att)
    
    return att
 
def save_photo(photo_id):
    params = {
        'v': API_VERSION,
        'access_token': ACCESS_TOKEN,
        'photos': photo_id,
        'photo_sizes': 1
    }
    photo = requests.get(
        'https://api.vk.com/method/photos.getById',
        params = params
    )
    r = requests.get(photo.json()['response'][0]["sizes"][-1]['src'], proxies=proxies)
    with open('img/%s.jpg' % photo.json()['response'][0]["id"], 'wb') as f:
        f.write(r.content)
 
 
def get_video_link(video_id):
    params = {
        'v': API_VERSION,
        'access_token': ACCESS_TOKEN,
        'videos': video_id
    }
    video = requests.get(
        'https://api.vk.com/method/video.get',
        proxies=proxies,
        params=params
    )
    return video.json()['response']["items"][0]['player']
 
 
def save_doc(doc_id):
    params = {
        'v': API_VERSION,
        'access_token': ACCESS_TOKEN,
        'docs': doc_id
    }
    doc = requests.get(
        'https://api.vk.com/method/docs.getById',
        proxies=proxies,
        params=params
    )
    r = requests.get(doc.json()['response'][0]["url"], proxies=proxies)
    with open('docs/%s.%s' % (doc.json()['response'][0]["title"], doc.json()['response'][0]["ext"]), 'wb') as f:
        f.write(r.content)
 
 
def save_json(data):
    with open('posts.json', 'w') as file:
        json.dump(data, file, indent = 2, ensure_ascii = False)
 
 
def save_csv(data):
    with open('posts_data.csv', 'a') as file:
        writer = csv.writer(file)
        writer.writerow((
            data['id'], 
            data['date'], 
            data['attachments'],
            data['text']
        ))
   
 
def to_json(post_dict):
    try:
        data = json.load(open('posts_data.json'))
    except:
        data = []
    
    data.append(post_dict)
    
    with open('posts_data.json', 'w') as file:
        json.dumo(data, file, indent = 2, ensure_ascii = False)
 
        
 
if __name__ == '__main__':
    main()

import vk_api
import json
import csv
import config as cnf
 
vk_session = vk_api.VkApi(cnf.login, cnf.password)
vk_session.auth()
vk = vk_session.get_api()
tools = vk_api.VkTools(vk_session)
 
def id_by_name(name):
    return vk.groups.getById(group_id = name)[0]['id']
 
 
def main():
    vk_session = vk_api.VkApi(cnf.login, cnf.password)
 
    try:
        vk_session.auth(token_only=True)
    except vk_api.AuthError as error_msg:
        print(error_msg)
        return
 
    tools = vk_api.VkTools(vk_session)
    wall = tools.get_all(
        'wall.get', 100, {'owner_id': cnf.owner_id}
    )
    print('Всего постов: ', wall['count'])
    items = wall['items']
    for post in items:
        print(post)
        get_data_post = get_data(post)
        save_csv(get_data_post)
        print('PPC')
 
def get_data(post):
    try:
        post_id = post['id']
    except:
        post_id = 'null'
 
    try:
        post_date = post['date']
    except:
        post_date = 'null'
 
    try:
        post_attachments = post['attachments']
    except:
        post_attachments = '* * *'
 
    try:
        post_text = post['text']
    except:
        post_text = '* * *'
 
    try:
        #many_photos = post['attachments'][0]['photos']
        # вот сюда нужно добавить
        # many_photos = и получить все фото размером:
        #original: x
        #big: r
        #normal: q
        #small: p
        #micro: s
    except:
        continue
 
    data = {
        'id': post_id,
        'date': post_date,
        'attachments': post_attachments,
        'text': post_text
    }
 
    return data
 
 
 
def save_csv(data):
    with open('posts_data.csv', 'a') as cfile:
        writer = csv.writer(cfile)
        writer.writerow((
            data['id'],
            data['date'],
            data['attachments'],
            data['text']
        ))
 
 
 
def save_json(data):
    with open('wall.json', 'w') as jfile:
        json.dump(data, jfile, indent=2, ensure_ascii=False)
 
 
 
if __name__ == '__main__':
    main()

import vk_api
import json
import csv
import config as cnf
 
vk_session = vk_api.VkApi(cnf.login, cnf.password)
vk_session.auth()
vk = vk_session.get_api()
tools = vk_api.VkTools(vk_session)
 
def id_by_name(name):
    return vk.groups.getById(group_id = name)[0]['id']
 
 
def main():
    vk_session = vk_api.VkApi(cnf.login, cnf.password)
 
    try:
        vk_session.auth(token_only=True)
    except vk_api.AuthError as error_msg:
        print(error_msg)
        return
 
    tools = vk_api.VkTools(vk_session)
    wall = tools.get_all(
        'wall.get', 1, {'owner_id': cnf.owner_id, 'offset': 1}
    )
    print('Всего постов: ', wall['count'])
    items = wall['items']
    for post in items:  #print(post)
        get_data_post = get_data(post)  #print(get_data_post)
        save_csv(get_data_post)  #print('PPC')
 
def get_data(post):
    print(post['attachments'])
    try:
        post_id = post['id']
    except:
        post_id = 'null'
 
    try:
        post_date = post['date']
    except:
        post_date = 'null'
 
    try:
        post_attachments = post['attachments']
        for attachment in post_attachments:
            if attachment['type'] == 'photo':
                photo_value = attachment['photo']
                sizes = photo_value['sizes']
                for size in sizes:
                    print(size['type'], size['url'], size['width'], size['height'])
 
    except:
        post_attachments = '* * *'
 
    try:
        post_text = post['text']
    except:
        post_text = '* * *'
 
    ##try:
        ###many_photos = post['attachments'][0]['photos']
        ### вот сюда нужно добавить
        ### many_photos = и получить все фото размером:
        ###original: x
        ###big: r
        ###normal: q
        ###small: p
        ###micro: s
    ##except:
        ##continue
 
    data = {
        'id': post_id,
        'date': post_date,
        'attachments': post_attachments,
        'text': post_text
    }
 
    return data
 
 
 
def save_csv(data):
    with open('posts_data.csv', 'a') as cfile:
        writer = csv.writer(cfile, delimiter=",")
        writer.writerow((
            data['id'],
            data['date'],
            data['attachments'],
            data['text']
        ))
 
 
 
def save_json(data):
    with open('wall.json', 'w') as jfile:
        json.dump(data, jfile, indent=2, ensure_ascii=False)
 
 
 
if __name__ == '__main__':
    main()

def flattenjson(d):
    out = []
    def flat(obj,keys='[',delim=']['):
        if type(obj) is dict:
            for k in obj:
                flat(obj[k], keys + repr(k) + delim)
        elif type(obj) is list:
            for i,x in enumerate(obj):
                flat(x, keys + repr(i) + delim)
        else:
            out.append(keys[:-1])
    flat(d)
    return out

with open('data.json') as f: 
    d = json.load(f)
l = flattenjson(d)
 
for i in l:
    if "photo" in i:
        print(i,eval('d'+i))