pythonでGCP(Google Cloud Platform)　~webスクレイピングの定期実行~

# Function dependencies, for example:
# package>=version
gspread>=5.2.0
pandas>=1.2.2
oauth2client>=4.1.3
google-cloud-error-reporting==0.30.0
selenium==3.141.0
setuptools

import time
import pytz
import os
import shutil
import stat
from pathlib import Path
from selenium import webdriver
import datetime
import pprint
import threading
from concurrent.futures import ThreadPoolExecutor
import gspread
import pandas as pd
from oauth2client.service_account import ServiceAccountCredentials

global driver


def add_execute_permission(path: Path, target: str = "u"):
    """Add `x` (`execute`) permission to specified targets."""
    mode_map = {
        "u": stat.S_IXUSR,
        "g": stat.S_IXGRP,
        "o": stat.S_IXOTH,
        "a": stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH,
    }

    mode = path.stat().st_mode
    for t in target:
        mode |= mode_map[t]

    path.chmod(mode)


def settingDriver():
    print("driver setting")
    global driver

    driverPath = "/tmp" + "/chromedriver"
    headlessPath = "/tmp" + "/headless-chromium"

    # copy and change permission
    print("copy headless-chromium")
    shutil.copyfile(os.getcwd() + "/headless-chromium", headlessPath)
    add_execute_permission(Path(headlessPath), "ug")

    print("copy chromedriver")
    shutil.copyfile(os.getcwd() + "/chromedriver", driverPath)
    add_execute_permission(Path(driverPath), "ug")

    chrome_options = webdriver.ChromeOptions()

    chrome_options.add_argument("--headless")
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--window-size=1280x1696")
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--hide-scrollbars")
    chrome_options.add_argument("--enable-logging")
    chrome_options.add_argument("--log-level=0")
    chrome_options.add_argument("--v=99")
    chrome_options.add_argument("--single-process")
    chrome_options.add_argument("--ignore-certificate-errors")
    chrome_options.add_argument("--disable-dev-shm-usage")

    chrome_options.binary_location = headlessPath

    print("get driver")
    driver = webdriver.Chrome(executable_path=driverPath, options=chrome_options)


# 評価総件数を取得
def get_num(path,total_num):
    global driver
    num_string = driver.find_element_by_xpath(path).get_attribute("aria-label")
    percent_float = float((num_string).replace("点で評価したユーザーが","").replace("%います","")[1:])
    num = total_num*(percent_float/100)
    return round(num)


def scraping_main(url):
    global driver
    return_inf = {}
    num_string = ""
    driver.get(url)
    time.sleep(1)
    num_string = driver.find_element_by_xpath("//*[@id='__next']/div[1]/div/div/main/section[1]/div[3]/div/div[1]/div[1]/div[2]/p").text
    # 未評価の場合はすべて0
    if "未評価" == num_string:
        print(4)
        return_inf["総評価件数"] = 0
        for i in range(5):
            return_inf["星{}の評価件数".format(5-i)] = 0
    else:
        total_num = float((num_string).replace("/ ","").replace("件","").replace(",",""))
        return_inf["総評価件数"] = round(total_num)
        # 星ごとの評価総件数を取得
        for i in range(5):
            path = "//*[@id='__next']/div[1]/div/div/main/section[1]/div[3]/div/div[1]/div[2]/div/div[{}]/span[2]".format(i+1)
            num = get_num(path,total_num)
            return_inf["星{}の評価件数".format(5-i)] = num
    print(return_inf)
    return return_inf


SCOPES = ["https://spreadsheets.google.com/feeds","https://www.googleapis.com/auth/drive"]
SERVICE_ACCOOUNT_FILE = "movie-evaluation-344802-f41fbd96a6ce.json"
SPREADSHEET_KEY = "1uBhVPvfON09jvsQC7GzyCn5W4Uzqn5_OtkFhxRIHAb4"

def main(request1,request2):
    settingDriver()
    dt_now = datetime.datetime.now(pytz.timezone('Asia/Tokyo'))
    credentials = ServiceAccountCredentials.from_json_keyfile_name(SERVICE_ACCOOUNT_FILE,SCOPES)
    gs = gspread.authorize(credentials)
    wb = gs.open_by_key(SPREADSHEET_KEY)

    ws = wb.worksheet("映画一覧")
    df = pd.DataFrame(ws.get_all_values())
    df.columns = df.iloc[0]
    df = df.drop(df.index[[0]])

    # シート「映画一覧」の映画数だけ情報抽出
    for index, row in df.iterrows():
        # 該当の映画のシートが未作成の場合作成する
        if wb.get_worksheet(index).title != row["映画タイトル"]:
            # シート「ひな形」をコピーし新規シートにする
            ws_base = wb.get_worksheet(-1)
            wb.duplicate_sheet(source_sheet_id = ws_base.id, new_sheet_name = row["映画タイトル"],insert_sheet_index = index)
        # スレッド処理で映画の評価情報取得
        pool = ThreadPoolExecutor(max_workers=1)
        future = pool.submit(scraping_main, row["URL"])
        evaluation_inf = future.result()
        pool.shutdown()
        # 総評価件数が0以外（未評価でない）のときのみ、評価を反映させる
        if 0 != evaluation_inf["総評価件数"]:
            ws = wb.get_worksheet(index)
            # 新規情報記入行の追加
            ws.add_rows(1)
            # スクレイピング結果反映
            datas = []
            datas.append(str(dt_now))
            datas.append(evaluation_inf["総評価件数"])
            datas.append(evaluation_inf["星5の評価件数"])
            datas.append(evaluation_inf["星4の評価件数"])
            datas.append(evaluation_inf["星3の評価件数"])
            datas.append(evaluation_inf["星2の評価件数"])
            datas.append(evaluation_inf["星1の評価件数"])
            ws.append_row(datas)
    print("driver quit")
    driver.quit()

gcloud functions deploy movie_evaluation 
--entry-point main 
--runtime python39 
--region asia-northeast1 
--trigger-topic=hourlate 
--allow-unauthenticated 
--memory 1GB

0  */1 * * *

import pytz
dt_now = datetime.datetime.now(pytz.timezone('Asia/Tokyo'))