Analyzing the EU Migration Crisis as reflected on Twitter¶
Jounral of Cartography and Geographic Information, Kartographische Nachrichten¶
Author Information
- Sagnik Mukherjee : sagnik.mukherjee1@tu-dresden.de ; https://orcid.org/0000-0001-8938-6154
- Eva Hauthal : eva.hauthal@tu-dresden.de ; https://orcid.org/0000-0001-8917-600X
- Dirk Burghardt : dirk.burghardt@tu-dresden.de ; https://orcid.org/0000-0003-2949-4887
Institute of Cartography, Technische Universität Dresden, 01069 Dresden, Germany
In [1]:
import warnings
warnings.filterwarnings('ignore')
from pathlib import Path
import pandas as pd
import geopandas as gp
from python_hll.hll import HLL
from python_hll.util import NumberUtil
from shapely.geometry import Point
import matplotlib.pyplot as plt
%matplotlib inline
import geoplot as gplt
import geoplot.crs as gcrs
import contextily as ctx
OUTPUT = Path.cwd().parents[0] / "Output"
df = pd.read_csv(Path.cwd().parents[0] /"CSV"/"HLL"/"ESM_3.csv",usecols = [0,1,2,3,4,5])
eu = gp.read_file(Path.cwd().parents[0]/ "Europe_coastline_shapefile" / "ESM_4.shp",crs = 'epsg:4326')
df['post_hll'] = df['post_hll'].apply(lambda x: x[2:])
df['date_hll'] = df['date_hll'].apply(lambda x: x[2:])
df['user_hll'] = df['user_hll'].apply(lambda x: x[2:])
df
Out[1]:
In [3]:
df[df['hashtag']=='refugees']
Out[3]:
In [2]:
#hll functions
def hll_from_byte(hll_set):
"""Return HLL set from binary representation"""
return HLL.from_bytes(
NumberUtil.from_hex(
hll_set, 0, len(hll_set)))
def union_hll(hll, hll2):
"""Union of two HLL sets. The first HLL set will be modified in-place."""
hll.union(hll2)
def union_all_hll(hll_series,cardinality = True):
"""HLL Union and (optional) cardinality estimation from series of hll sets
Args:
hll_series: Indexed series (bins) of hll sets.
cardinality: If True, returns cardinality (counts). Otherwise,
the unioned hll set will be returned.
"""
hll_set = None
for hll_set_str in hll_series.values.tolist():
if hll_set is None:
# set first hll set
hll_set = hll_from_byte(hll_set_str)
continue
hll_set2 = hll_from_byte(hll_set_str)
union_hll(hll_set, hll_set2)
return hll_set.cardinality()
In [3]:
#union,intersection plot functions
def extract_tags(htag,keep_column):
column_list = ['post_hll','user_hll','date_hll']
column_list.remove(f'{keep_column}')
mask =f"hashtag == '{htag}'"
df_htag = df.query(mask)
df_htag.drop(columns = column_list,inplace =True)
return df_htag
def three_df(tag_a,tag_b,tag_c,column):
df_a = extract_tags(f'{tag_a}',f'{column}')
df_b = extract_tags(f'{tag_b}',f'{column}')
df_c = extract_tags(f'{tag_c}',f'{column}')
calculate_post_total(df_a,df_b,df_c)
set_operations(df_a,df_b,df_c)
def calculate_post_total(df_a,df_b,df_c):
dfs = {
f"{TAG_A}" : df_a,
f"{TAG_B}": df_b,
f"{TAG_C}" :df_c
}
ptotal = {}
for hashtag, dfs in dfs.items():
# drop bins with no values
cardinality_total = union_all_hll(
dfs[f"{COLUMN}"].dropna())
ptotal[hashtag] = cardinality_total
print(
f"{ptotal[hashtag]} distinct posts "
f"used {hashtag.upper()}")
d_common(df_a,df_b,df_c,ptotal)
def d_common(df_a,df_b,df_c,ptotal):
union_a_b = pd.concat([df_a, df_b])
union_a_c = pd.concat([df_a, df_c])
union_b_c = pd.concat([df_c, df_b])
dfs = {
f"{TAG_A}_{TAG_C}": union_a_c,
f"{TAG_A}_{TAG_B}": union_a_b,
f"{TAG_B}_{TAG_C}": union_b_c
}
distinct_common = {}
for hashtag_set, dfs in dfs.items():
cardinality = union_all_hll(
dfs[f"{COLUMN}"].dropna())
distinct_common[hashtag_set] = cardinality
print(
f"{distinct_common[hashtag_set]} distinct total posts "
f"which had either {hashtag_set.split('_')[0]} "
f"or {hashtag_set.split('_')[1]} (union)")
distinct_intersection = {}
for a, b in [(f"{TAG_A}", f"{TAG_B}"), (f"{TAG_A}", f"{TAG_C}"), (f"{TAG_B}", f"{TAG_C}")]:
a_total = ptotal[a]
b_total = ptotal[b]
common_tags = f'{a}_{b}'
intersection_count = a_total + b_total - distinct_common[common_tags]
distinct_intersection[common_tags] = intersection_count
print(
f"{distinct_intersection[common_tags]} distinct posts "
f"with hashtags with {a} and {b} (intersection)")
union_a_b_c = pd.concat([df_a, df_b, df_c])
cardinality = union_all_hll(
union_a_b_c[f"{COLUMN}"].dropna())
union_count_all = cardinality
intersection_count_all = union_count_all - \
ptotal[TAG_A] - \
ptotal[TAG_B] - \
ptotal[TAG_C] + \
distinct_intersection[f'{TAG_A}_{TAG_B}'] + \
distinct_intersection[f'{TAG_A}_{TAG_C}'] + \
distinct_intersection[f'{TAG_B}_{TAG_C}']
print(f'Union Count : {union_count_all}',
f'Intersection Count : {intersection_count_all}')
def make_tuple(lat,lon):
return (lat,lon)
def make_lists(dfx):
return dfx.apply(lambda x: make_tuple(x.latitude,x.longitude),axis=1).tolist()
def set_operations(df_a,df_b,df_c):
set_a = set(make_lists(df_a))
set_b = set(make_lists(df_b))
set_c = set(make_lists(df_c))
union_final = set_a.union(set_b,set_c)
intersect_foo = set(set_a).intersection(set_b)
intersect_final = set(set_c).intersection(intersect_foo)
makeGeoSeries(union_final,intersect_final)
def makeGeoSeries(union, intersect):
union_foo = [Point(coord[1], coord[0]) for coord in union]
union_points = gp.GeoSeries(union_foo)
union_points.set_crs(epsg =3857,inplace =True)
intersect_foo = [Point(coord[1], coord[0]) for coord in intersect]
intersect_points = gp.GeoSeries(intersect_foo)
intersect_points.set_crs(epsg =3857,inplace =True)
plot(union_points,intersect_points)
def plot(u_points,i_points):
union = u_points
intersection = i_points
fig,axes = plt.subplots(ncols =2,
nrows =1,
figsize=(20,10),
subplot_kw={'projection': gcrs.WebMercator()}
)
gplt.webmap(eu, projection=gcrs.WebMercator(),
provider = ctx.providers.Stamen.TonerLite,
ax =axes[0]
)
gplt.kdeplot(union, ax=axes[0],n_levels=15, cmap='turbo')
axes[0].set_title("KDE of union function points",fontsize =10)
gplt.webmap(eu, projection=gcrs.WebMercator(),
provider = ctx.providers.Stamen.TonerLite,
ax =axes[1]
)
gplt.kdeplot(intersection, ax=axes[1],n_levels=15, cmap='turbo')
axes[1].set_title("KDE of intersection function points",fontsize =10)
plt.savefig(OUTPUT / 'USERHLL.jpeg',quality =95,bbox_inches = 'tight')
In [4]:
%%time
TAG_A ='refugees'
TAG_B = 'refugeeswelcome'
TAG_C = 'migrants'
COLUMN = "user_hll"
three_df(TAG_A,TAG_B,TAG_C,COLUMN)
For further Information, please check the following links:
HLL Operations with YFCC 100m dataset : https://ad.vgiscience.org/yfcc_gridagg/01_preparations.html
HLL Operations with Instagram Sunrise-Sunset dataset : https://wwwpub.zih.tu-dresden.de/~s7398234/vis/instagram/Instagram_sunrise_sample.html
General Information on LBSN: https://lbsn.vgiscience.org/