World equity markets valuation

# -*- coding: utf-8 -*-

import urllib.request
import re
from collections import defaultdict
import pandas as pd

TARGETS = ['<div class="float-left in-left col-priceBook',
           '<div class="float-left in-right col-priceEarnings',
           '<div class="float-left in-left col-numHoldings',
           '<div class="float-left in-right col-beta3Yr',
           '<div class="float-left in-left col-thirtyDaySecYield',
           '<div class="float-left in-right col-standardDeviation3Yr',
           '<div class="float-left in-left col-twelveMonTrlYld']


GROUPS= {
 'Materials': 'Industrials',
 'Communication':'Telecommunications',
 'Insurance': 'Financials',
 'Diversified Financials':  'Financials',
 'Capital Goods':'Industrials',
 'Banks' : 'Financials',
 'Consumer Services': 'Consumer Discretionary',
 'Transportation': 'Utilities',
 'Food Bevg Tobacco': 'Consumer Staples'}



board_categories = ['Consumer Discretionary',
 'Information Technology',
 'Consumer Staples',
 'Financials',
 'Industrials',
 'Energy',
 'Utilities',
 'Real Estate',
 'Health Care',
 'Telecommunications']

def locate(target, html):
    
    
    
    for i, line in enumerate(html):
        if target in line:
            print ('line {} \n target {}'.format(line, target))
            print ()
            return i
    else:
        print ('cannot find {}'.format(target))
        return None
    
def extract(i, html):
    # print (html[i:i+10])
    return html[i+2], html[i+5], html[i+9]    


def parse_javascript(x):
    
    start_ind = []
    end_ind = []
    
    
    ranking = defaultdict(list)
    
    for j, c in enumerate(x):
        if '{'  == c:
            start_ind.append(j)
        elif '}' == c:
            end_ind.append(j)
            
    assert len(start_ind) == len(end_ind)        
    for s,e in zip(start_ind, end_ind):
        # print (html[i][s:e])
        items = x[s:e].split(',')
        
        value = items[0].split(':')[-1]
        value = re.findall('\w+', value)
        ranking['category'].append(' '.join(value))
        
        value = items[1].split(':')[-1]
        ranking['percentage'].append(float(re.findall('\d+\.\d+', value)[0]))
        ranking['rank'].append(int(items[2].split(':')[-1]))
    
        
    
    print ('found %d categories' %len(start_ind))
    
    return ranking


if __name__ == "__main__":
    

    with open('weblinks.txt', 'r') as f:
        links  = f.readlines()
    
    
    
    data = defaultdict(list) 
    
    categories = []
     
    for link in links:
        link = link.rstrip('\n')
        country = link.split('/')
        
        myURL = urllib.request.urlopen(link)
        html = myURL.read()
        html = html.decode('utf-8')
        html = html.split('\n')
        
        target = 'var tabsSectorDataTable =[{"name":'
        i = locate(target, html)
        
        ranking = parse_javascript(html[i])
        
        # find all possible categories
        # for category in ranking['category']:
        #     if category not in categories:
        #         categories.append(category)
                

        
        for j, target in enumerate(TARGETS):
    
            i = locate(target, html)
            
            metric, date, value = extract(i, html)
            
            if j == 0:
                data['country'].append(country)
                data['dates'].append(date)
                
            data[metric].append(value)
            
        
        percentages = defaultdict(float)
        
        for c, p  in zip(ranking['category'], ranking['percentage']):
            
            if c in GROUPS:
                c = GROUPS[c]
                
            if c in board_categories:
                percentages[c] += p
                    
                
        print ('check sum: ', sum(percentages.values()))
        
        
        for category in board_categories:
            data[category].append(percentages[category])
                


    
    for key, value in data.items():
        print (key, len(value))
    
    table = pd.DataFrame.from_dict(data)
        
    table.to_csv('etf_Statistics.csv')  
        

import pandas as pd
import geopandas
import numpy as np

# import folium
import matplotlib.pyplot as plt
from shapely.geometry import Polygon

def str_float(x):
    return float(x[0:-1])

table = pd.read_csv('etf_Statistics.csv')

for c in ['30 Day SEC Yield',
       'Standard Deviation (3y)',
       '12m Trailing Yield']:
    
    table[c] = table[c].apply(str_float)
    



# Read the geopandas dataset
world = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres'))
df = pd.DataFrame(
    {'name': ['Hong Kong', 'Singapore'],
     'geometry': [Polygon([(111.177216, 19.302711),
                           (117.177216, 19.302711),
                           (117.177216, 25.302711),
                           (111.177216, 25.302711)]),
                  Polygon([(102.177216, 2.302711),
                           (104.177216, 2.302711),
                           (104.177216, 0.302711),
                           (102.177216, 0.302711)])
                  ]})

world = pd.concat((world, df))

print(world.head())


names = world.name.tolist()


translation = {}

for c in table['country']:
    # print ('------------------')
    for name in names:
        name2 = name.lower()
        name2 = name2.replace(' ', '-')
        
        if name2 in c:
            translation[c] = name
            break
        
    else:
        print (c)
        
# have to specify the country's name for some        
for c in table['country']:
    
    if 'sp-500' in c:
        translation[c] = 'United States of America'
        
    if 'uae' in c:
        translation[c] = 'United Arab Emirates'
    if '287286' in c:
        translation[c] = 'Argentina'
      
ind = [i for i, t in enumerate(table['country']) if t in translation]        
        
table = table.iloc[ind,:]
        
table['country'] = [translation[i] for i in table['country']]

table2 = world.merge(table, how="left", left_on=['name'], right_on=['country'])


plt.close('all')
for c in table2.columns[9:]:
    # plt.figure()
    with plt.style.context(("seaborn", "ggplot")):
        ax = table2.plot(c,
                    figsize=(15,10),
                    edgecolor="black",
                    legend=True,
                    legend_kwds={"label": c, "orientation":"horizontal"},
                    cmap=plt.cm.seismic,
                    vmin=0,
                    vmax = np.nanmean(table2[c]) * 2,
                    missing_kwds={
                                    "color":"grey",
                                    "edgecolor":"black",
                                    "hatch":"---",
                                    "label":"Missing Values"
                                }
                )
        ax.set_axis_off()
        plt.title(c + ' ' + table2['dates'][np.invert(table2['dates'].isna())].iloc[0])
        
        c = c.replace('/', '_')
        c = c.replace(' ', '_')
        plt.savefig (c + ".png", transparent=True)