Python Flask CORS ,Crawling

Python에서 Flask를 사용하는 경우 CORS(Cross Origin Resource Sharing) 설정이 필요할 수 있습니다. 예를들어 다른 도메인이나 로컬 환경에서 자바스크립트로 api 등을 호출하는 경우 브라우저에서 동일 출처 위반의 에러가 나타날 수 있습니다. 이를 해결하려면 어떻게 할까요?

이때 CORS를 설정할 수 있습니다. CORS는 자바스크립트를 사용한 api 등의 리소스 호출시 동일 출처(같은 호스트네임)가 아니더라도 정상적으로 사용 가능하도록 도와주는 방법입니다.

 

http://localhost:5000/  이렇게 접근하면 접근을 하지 못한다. 그래서 cors설정을 통해 접근할 수 있게 한다.

=>가지고 오지 못함

=>CORS에러 

cmd에 설치하고

from flask_cors.extension import CORS

app = Flask(__name__,static_url_path='')
CORS(app)

이렇게 써주기

---

크롤링 하기 

day14의 myrequest.py실행하고 ...

import requests

from bs4 import BeautifulSoup

htmls =requests.get("http://127.0.0.1:5000/")
print(htmls.text)

soup = BeautifulSoup(htmls.text, "html.parser")
print("---------------------------------------")
print(soup)
print("---------------------------------------")
print(soup.find_all("tr"))

trs =soup.find_all("tr")

for idx,tr in enumerate(trs):
    if idx >0:
        tds =tr.find_all("td")
        print(tds[1].text,tds[3].text)

select 방식 

trs2 =soup.select('tr')
for idx,tr in enumerate(trs2):
    if idx >0:
        tds =tr.select("td")
        print(tds[1].text,tds[3].text)

---

웹크롤링하여 db에 넣기 

import requests
from bs4 import BeautifulSoup
import datetime
import pymysql

conn = pymysql.connect(host='localhost', user='root', password='python',
                       db='python',port=3305, charset='utf8')
curs = conn.cursor()

now = datetime.datetime.now()
ymd = now.strftime("%Y%m%d.%H%M")
print(ymd)


htmls = requests.get('https://vip.mk.co.kr/newSt/rate/item_all.php?koskok=KOSDAQ&orderBy=dd')
#https://vip.mk.co.kr/newSt/rate/item_all.php?koskok=KOSDAQ&orderBy=dd
#https://vip.mk.co.kr/newSt/rate/item_all.php

htmls.encoding = "euc-kr"

soup = BeautifulSoup(htmls.text, "html.parser")

st2s = soup.select(".st2")

for st in st2s:
    s_name = st.text
    s_code = st.select("a")[0]['title']
    price = st.parent.select("td")[1].text.replace(",","")
    
    print(s_name,s_code,price,ymd)
    sql = f"""insert into stock (s_code,s_name,price,ymd) 
        values ('{s_code}','{s_name}','{price}','{ymd}')"""

    print("sql",sql)
    cnt = curs.execute(sql)
    print("cnt",cnt)

conn.commit()
curs.close() 
conn.close()

---

Selenium 모듈을 이용하는 방법 (동적 크롤링)

크롬 정보와 맞게 다운로드 

from selenium import webdriver

# Webdriver 실행
browser = webdriver.Chrome()
browser.get('http://127.0.0.1:5000/')

trs = browser.find_elements_by_css_selector('#tb > tr')

for tr in trs:
    tds = tr.find_elements_by_css_selector("td")
    print(tds[1].text,tds[3].text)

from selenium import webdriver
from selenium.webdriver.common.by import By

# Webdriver 실행
browser = webdriver.Chrome()
browser.get('http://127.0.0.1:5000/')

trs = browser.find_elements(by=By.CSS_SELECTOR, value = "#tb > tr")


for tr in trs :
    tds = tr.find_elements(by=By.CSS_SELECTOR, value = "td")
    print(tds[1].text, tds[3].text)

=>하지만 이런식으로 쓰는것이 좋음

 

---

import pymysql

class DaoMango:
    def __init__(self):
        self.conn = pymysql.connect(host='localhost', user='root', password='python',
                               db='python',port=3305, charset='utf8')
        self.curs = self.conn.cursor(pymysql.cursors.DictCursor)

    
    def myinsert(self,m_name,menu):
        sql = f"""insert into mangoplate (m_name,menu) 
                values ('{m_name}','{menu}')"""
                
        print("sql",sql)
        
        cnt = self.curs.execute(sql)
        self.conn.commit()
        return cnt

    
    def __del__(self):
        self.curs.close()
        self.conn.close()
    
if __name__ == '__main__':
    de = DaoMango()
    cnt = de.myinsert('1','1')
    print(cnt)

import sys
from PyQt5 import uic
from PyQt5.QtWidgets import QApplication, QMainWindow
from selenium import webdriver
from selenium.webdriver.common.by import By
from day14.mymango_dao import DaoMango

form_class = uic.loadUiType("mymango_gui.ui")[0]

class MainClass(QMainWindow, form_class):
    def __init__(self) :
        QMainWindow.__init__(self)
        self.browser = webdriver.Chrome()
        self.browser.get("https://www.mangoplate.com/search/%EB%8C%80%EC%A0%84%20%EC%98%A4%EB%A5%98%EB%8F%99")
        self.dm = DaoMango()
        self.setupUi(self)
        self.show()
        self.pb.clicked.connect(self.myclick)
        self.pb_scrap.clicked.connect(self.myscrap)
    
    def myclick(self):
        url = self.le.text()
        self.browser.get(url)
        
    def myscrap(self):
        figs = self.browser.find_elements(by=By.CSS_SELECTOR, value='figcaption')
        for fig in figs:
            try:
                m_name = fig.find_elements(by=By.CSS_SELECTOR, value='a')[0].text
                if m_name != "":
                
                    menu = fig.find_elements(by=By.CSS_SELECTOR, value='.etc')[0].find_elements(by=By.CSS_SELECTOR, value='span')[0].text
                    print(":",m_name,":",menu)
                
                    cnt = self.dm.myinsert(m_name, menu)
                    print(cnt)
            except:
                print("error:")
            
    
        
if __name__ == "__main__" :
    app = QApplication(sys.argv) 
    window = MainClass() 
    app.exec_()

from selenium import webdriver
from selenium.webdriver.common.by import By

# 102.0.5005.63
 
browser = webdriver.Chrome()
browser.get("https://www.mangoplate.com/search/%EB%8C%80%EC%A0%84%20%EC%98%A4%EB%A5%98%EB%8F%99")

 
trs = browser.find_elements(by=By.CSS_SELECTOR, value='#tb > tr')

for tr in trs:
    tds = tr.find_elements(by=By.CSS_SELECTOR, value='td')
    print(tds[1].text,"\t",tds[3].text)