更新任务获取接口
This commit is contained in:
57
main.py
57
main.py
@@ -3,60 +3,65 @@ from spider.task import *
|
||||
from logger import error_logger, record_full_log
|
||||
|
||||
TASK_TYPE = {
|
||||
0: post,
|
||||
'get_account_profile': playwright_get_user_profile,
|
||||
'check_account_cookies': playwright_check_account_cookies,
|
||||
'post': playwright_post
|
||||
}
|
||||
|
||||
HOST = "http://118.193.40.152:8091"
|
||||
HOST = "http://192.168.1.69:8001"
|
||||
|
||||
|
||||
def get_task(platform):
|
||||
# if lock._block.locked():
|
||||
# return
|
||||
response = requests.get(f'{HOST}/services/task', json={'platform': platform})
|
||||
result = response.json()
|
||||
if result['code'] == 0:
|
||||
def get_task():
|
||||
if lock._block.locked():
|
||||
return
|
||||
url = f'{HOST}/queue/get-data'
|
||||
header = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
response = requests.post(url, headers=header)
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
return result
|
||||
|
||||
|
||||
def task_callback(task_type, queue_id, data, status=1, err_msg=''):
|
||||
def task_callback(tid, data, status='success', msg='success'):
|
||||
body = {
|
||||
'id': tid,
|
||||
'status': status,
|
||||
'data': data,
|
||||
'message': msg,
|
||||
}
|
||||
response = requests.post(
|
||||
f'{HOST}/services/task/callback',
|
||||
json={
|
||||
'task_type': task_type,
|
||||
'queue_id': queue_id,
|
||||
'status': status,
|
||||
'data': data,
|
||||
'err_msg': err_msg,
|
||||
}
|
||||
f'{HOST}/queue/handle-data',
|
||||
json=body
|
||||
)
|
||||
result = response.json()
|
||||
if result['code'] != 0:
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(f"任务回调失败:{result['msg']}")
|
||||
|
||||
|
||||
def execute_task(queue_id, task_type, **kwargs):
|
||||
def execute_task(tid, task_type, **kwargs):
|
||||
try:
|
||||
result = TASK_TYPE.get(task_type)(**kwargs)
|
||||
task_callback(task_type, queue_id, data=result)
|
||||
task_callback(tid, data=result)
|
||||
except AuthException as e:
|
||||
record_full_log(error_logger, e)
|
||||
task_callback(task_type, queue_id, data={}, status=4, err_msg=str(e))
|
||||
task_callback(tid, data={}, status='fail', msg=str(e))
|
||||
except Exception as e:
|
||||
record_full_log(error_logger, e)
|
||||
task_callback(task_type, queue_id, data={}, status=0, err_msg=str(e))
|
||||
task_callback(tid, data={}, status='fail', msg=str(e))
|
||||
|
||||
|
||||
def main():
|
||||
with ThreadPoolExecutor(max_workers=1) as t:
|
||||
while True:
|
||||
try:
|
||||
task = get_task(0)
|
||||
task = get_task()
|
||||
if task is None:
|
||||
time.sleep(10)
|
||||
continue
|
||||
|
||||
task['data']['queue_id'] = task['queue_id']
|
||||
task['data']['task_type'] = task['task_type']
|
||||
task['data']['tid'] = task['id']
|
||||
task['data']['task_type'] = task['taskType']
|
||||
t.submit(execute_task, **task['data'])
|
||||
except Exception as e:
|
||||
error_logger.error(f'Main Error: {e}')
|
||||
|
||||
Reference in New Issue
Block a user