from konlpy.tag import Okt

tokenizer = Okt()
tokens = tokenizer.pos("아버지 가방에 들어가신다.", norm=True, stem=True)
print(tokens)

with open('raw_pos_tagged.pkl', 'wb') as f:
    pickle.dump(raw_pos_tagged, f) 

with open('raw_pos_tagged.pkl','rb') as f:
    data = pickle.load(f)

df = pd.read_excel('result_220328_1314.xlsx')
articles = df['Article'].tolist()
articles = ''.join(articles)

from konlpy.tag import Okt

tokenizer = Okt()
raw_pos_tagged = tokenizer.pos(articles, norm=True, stem=True)

word_cleaned = ['불용어가 제거된 단어 목록']

# NLTK의 Text() 클래스에서 matplotlib의 plot 기능 제공
word_counted = nltk.Text(word_cleaned)
plt.figure(figsize=(15, 7))
word_counted.plot(50)

import requests
from bs4 import BeautifulSoup

import pandas as pd
from datetime import datetime
import time # time.sleep()
import re

https://search.naver.com/search.naver?
    where=news&
    sm=tab_jum& <!-- 불필요 -->
    query=데이터분석

query = input() # 데이터분석

url = f'https://search.naver.com/search.naver?where=news&query={query}'
web = requests.get(url).content
source = BeautifulSoup(web, 'html.parser')

news_subjects = source.find_all('a', {'class' : 'news_tit'})

subject_list = []

for subject in news_subjects:
    subject_list.append(subject.get_text())

urls_list = []

for urls in source.find_all('a', {'class' : 'info'}):
    if urls.attrs['href'].startswith('https://news.naver.com'):
        urls_list.append(urls.attrs['href'])

web_news = requests.get(urls_list[0]).content
source_news = BeautifulSoup(web_news, 'html.parser')

ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

headers = {'User-Agent':'Mozilla/5.0
           (Windows NT 6.3; Win64; x64) 
           AppleWebKit/537.36 (KHTML, like Gecko)
           Chrome/63.0.3239.132 Safari/537.36'}

web_news = requests.get(urls_list[0], headers=headers).content
source_news = BeautifulSoup(web_news, 'html.parser')

title = source_news.find('h3', {'id' : 'articleTitle'}).get_text()
date = source_news.find('span', {'class' : 't11'}).get_text()

# 2022.03.25. 오전 10:18
date = source_news.find('span', {'class' : 't11'}).get_text()

# 2022.03.25.10:18am
pd_date = pd.Timestamp(reformatted_date)

article = source_news.find('div', {'id' : 'articleBodyContents'}).get_text()

article = article.replace("\n", "")
article = article.replace("// flash 오류를 우회하기 위한 함수 추가function _flash_removeCallback() {}", "")
article = article.replace("동영상 뉴스       ", "")
article = article.replace("동영상 뉴스", "")
article = article.strip()

press_company = source_news.find('address', {'class' : 'address_cp'}).find('a').get_text()
print(press_company)

for url in urls_list:
    ...

    titles.append(title)
    dates.append(date)
    articles.append(article)
    article_urls.append(url)
    press_companies.append(press_company)

article_df = pd.DataFrame({'Title':titles, 
                           'Date':dates, 
                           'Article':articles, 
                           'URL':article_urls, 
                           'PressCompany':press_companies})

from sklearn import datasets, linear_model, model_selection, metrics

data_total = datasets.load_boston()

x = data_total.data
y = data_total.target

train_x, test_x, train_y, test_y =
    model_selection.train_test_split(x, y, test_size=0.3)

# 학습 전의 모델 생성
model = linear_model.LinearRegression()
# 모델에 학습 데이터를 넣으면서 학습 진행
model.fit(train_x, train_y)

# 모델에게 새로운 데이터를 주면서 예측 요구
predictions = model.predict(test_x)
# 예측 결과를 바탕으로 성능 점수 확인
metrics.mean_squared_error(predictions, test_y)

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

corpus = [doc1, doc2, doc3]

vectorizer = TfidfVectorizer()

X = vectorizer.fit_transform(corpus).todense()
# vertorizer.fit(corpus)
# X = vectorizer.trasnform(corpus)

[[0.0071001  0.00332632 0.         ... 0.         0.00166316 0.        ]
 [0.00889703 0.         0.00138938 ... 0.00138938 0.         0.00138938]]

import nltk
from nltk.corpus import stopwords
from collections import Counter

stop_words = stopwords.words("english")
stop_words.append(',')
stop_words.append('.')
stop_words.append('’')
stop_words.append('”')
stop_words.append('—')

file = open('movie_review.txt', 'r', encoding="utf-8")
lines = file.readlines()

tokens = []
for line in lines:
    tokenized = nltk.word_tokenize(line)
    for token in tokenized:
        if token.lower() not in stop_words:
            tokens.append(token)

tags = nltk.pos_tag(tokens)

word_list = []
for word, tag in tags:
    if tag.startswith('N'):
        word_list.append(word.lower())

counts = Counter(word_list)
print(counts.most_common(10))

import nltk

nltk.download()

sentence = 'NLTK is a leading platform for building Python programs to work with human language data. It provides easy-to-use interfaces to over 50 corpora and lexical resources such as WordNet, along with a suite of text processing libraries for classification, tokenization, stemming, tagging, parsing, and semantic reasoning, wrappers for industrial-strength NLP libraries, and an active discussion forum.'

nltk.word_tokenize(sentence)

from bs4 import BeautifulSoup 
from urllib.request import urlopen 

# 찾는 단어 입력
word = 'happiness'

url = f'https://alldic.daum.net/search.do?q={word}'
web = urlopen(url)
web_page = BeautifulSoup(web, 'html.parser')

text_search = web_page.find('span', {'class': 'txt_emph1'})
print(f'찾는 단어: {text_search.get_text()}')

list_search = web_page.find('ul', {'class': 'list_search'})
list_text = list_search.find_all('span', {'class': 'txt_search'})
definitions = [definition.get_text() in list_text]
print(f'단어의 뜻: {definitions}')

찾는 단어: happiness
단어의 뜻: ['행복', '만족', '기쁨', '행운']

# 찾는 영화 번호 입력 (향후 영화 제목으로 검색 구현)
movie = 208077

url = f'https://movie.naver.com/movie/search/result.naver?query={movie}'
web = urlopen(url)
web_page = BeautifulSoup(web, 'html.parser')

title = web_page.find('h3', {'class':'h_movie'}).find('a')
print(f'Movie Title: {title.get_text()}')

url = f'https://movie.naver.com/movie/bi/mi/detail.naver?query={movie}'
web = urlopen(url)
web_page = BeautifulSoup(web, 'html.parser')

director = web_page.find('div', {'class':'dir_product'}).find('a')
print(f'Director: {director.get_text()}')

actor_list = web_page.find('ul', {'class':'lst_people'})
actor_names = actor_list.find_all('a', {'class':'k_name'})
actors = [actor.get_text() for actor in actor_names]
print(f'Actors: {actors}')

import json

# 한국의 지도 데이터 참조
# @ https://github.com/southkorea/southkorea-maps
geo_path = 'skorea_municipalities_geo_simple.json'
geo_str = json.load(open(geo_path, encoding='utf-8'))

from pyprnt import prnt

prnt(geo_str, truncate=True, width=80)

import seaborn as sns

sns.heatmap(gu_df[])

matplotlib inline

# Windows
font_name = font_manager.FontProperties(fname="C:/~/malgun.ttf").get_name()
rc('font', family=font_name)

# Mac
rc('font', family='AppleGothic')

# 시각화된 데이터에서 열방향(axis=0)을 기준으로 인구수 데이터를 나눔 (인구 10만 단위)
crime_ratio = crime_count_norm.div(gu_df['인구수'], axis=0) * 100000

crime_ratio['전체발생비율'] = crime_ratio.mean(axis=1)

plt.figure(figsize = (10,10))

sns.heatmap(crime_ratio.sort_values(by='전체발생비율', ascending=False),
            annot=True, fmt='f', linewidths=.5, cmap='Reds')
plt.title('범죄 발생(전체발생비율로 정렬) - 각 항목을 정규화한 후 인구로 나눔')
plt.show()

# gu_df는 실습 자료에 서울시 경찰청의 소속 구 데이터를 추가한 DataFrame
gu_df['강간검거율'] = gu_df['강간(검거)']/gu_df['강간(발생)']*100
gu_df['강도검거율'] = gu_df['강도(검거)']/gu_df['강도(발생)']*100
gu_df['살인검거율'] = gu_df['살인(검거)']/gu_df['살인(발생)']*100
gu_df['절도검거율'] = gu_df['절도(검거)']/gu_df['절도(발생)']*100
gu_df['폭력검거율'] = gu_df['폭력(검거)']/gu_df['폭력(발생)']*100
gu_df['검거율'] = gu_df['소계(검거)']/gu_df['소계(발생)']*100

# 검거율에 해당되는 열의 집합 columns
columns = ['강간검거율', '강도검거율', '살인검거율', '절도검거율', '폭력검거율']

for row_index, row in gu_df_rate.iterrows():
    for column in columns:
        if row[column] > 100:
            gu_df.at[row_index, column] = 100

gu_df[ gu_df[columns] > 100 ] = 100