In [1]:
import sys; sys.path.append(_dh[0].split("knowknow")[0])
from knowknow import *

wos_base = "G:/My Drive/projects/qualitative analysis of literature/pre 5-12-2020/009 get everything from WOS"
basedir = Path(wos_base)
In [3]:
from csv import DictReader
In [11]:
# This cell ensures there are not overflow errors while importing large CSVs

import sys
import csv
maxInt = sys.maxsize

while True:
    # decrease the maxInt value by factor 10
    # as long as the OverflowError occurs.

    try:
        csv.field_size_limit(maxInt)
        break
    except OverflowError:
        maxInt = int(maxInt/10)
In [53]:
journal_keep = ["ETHNIC AND RACIAL STUDIES", "LAW & SOCIETY REVIEW", "DISCOURSE & SOCIETY", "SOCIOLOGICAL INQUIRY", "CONTRIBUTIONS TO INDIAN SOCIOLOGY", "SOCIETY & NATURAL RESOURCES", "RATIONALITY AND SOCIETY", "DEVIANT BEHAVIOR", "ACTA SOCIOLOGICA", "SOCIOLOGY-THE JOURNAL OF THE BRITISH SOCIOLOGICAL ASSOCIATION", "WORK EMPLOYMENT AND SOCIETY", "SOCIOLOGICAL METHODS & RESEARCH", "SOCIOLOGICAL PERSPECTIVES", "JOURNAL OF MARRIAGE AND FAMILY", "WORK AND OCCUPATIONS", "JOURNAL OF CONTEMPORARY ETHNOGRAPHY", "THEORY AND SOCIETY", "POLITICS & SOCIETY", "SOCIOLOGICAL SPECTRUM", "RACE & CLASS", "ANTHROZOOS", "LEISURE SCIENCES", "COMPARATIVE STUDIES IN SOCIETY AND HISTORY", "SOCIAL SCIENCE QUARTERLY", "MEDIA CULTURE & SOCIETY", "SOCIOLOGY OF HEALTH & ILLNESS", "SOCIOLOGIA RURALIS", "SOCIOLOGICAL REVIEW", "TEACHING SOCIOLOGY", "BRITISH JOURNAL OF SOCIOLOGY", "JOURNAL OF THE HISTORY OF SEXUALITY", "SOCIOLOGY OF EDUCATION", "SOCIAL NETWORKS", "ARMED FORCES & SOCIETY", "YOUTH & SOCIETY", "POPULATION AND DEVELOPMENT REVIEW", "SOCIETY", "JOURNAL OF HISTORICAL SOCIOLOGY", "HUMAN ECOLOGY", "INTERNATIONAL SOCIOLOGY", "SOCIAL FORCES", "EUROPEAN SOCIOLOGICAL REVIEW", "JOURNAL OF HEALTH AND SOCIAL BEHAVIOR", "SOCIOLOGICAL THEORY", "SOCIAL INDICATORS RESEARCH", "POETICS", "HUMAN STUDIES", "SOCIOLOGICAL FORUM", "AMERICAN SOCIOLOGICAL REVIEW", "SOCIOLOGY OF SPORT JOURNAL", "SOCIOLOGY OF RELIGION", "JOURNAL OF LAW AND SOCIETY", "GENDER & SOCIETY", "BRITISH JOURNAL OF SOCIOLOGY OF EDUCATION", "LANGUAGE IN SOCIETY", "AMERICAN JOURNAL OF ECONOMICS AND SOCIOLOGY", "ANNALS OF TOURISM RESEARCH", "SOCIAL PROBLEMS", "INTERNATIONAL JOURNAL OF INTERCULTURAL RELATIONS", "SOCIAL SCIENCE RESEARCH", "SYMBOLIC INTERACTION", "JOURNAL OF LEISURE RESEARCH", "ECONOMY AND SOCIETY", "INTERNATIONAL JOURNAL OF COMPARATIVE SOCIOLOGY", "SOCIAL COMPASS", "SOCIOLOGICAL QUARTERLY", "JOURNAL OF MATHEMATICAL SOCIOLOGY", "AMERICAN JOURNAL OF SOCIOLOGY", "REVIEW OF RELIGIOUS RESEARCH", "RURAL SOCIOLOGY", "JOURNAL FOR THE SCIENTIFIC STUDY OF RELIGION", "ARCHIVES EUROPEENNES DE SOCIOLOGIE", "CANADIAN JOURNAL OF SOCIOLOGY-CAHIERS CANADIENS DE SOCIOLOGIE"]
journal_keep = [x.lower() for x in journal_keep]
In [54]:
def filter_rows(rows):
    for r in rows:
        if r['DT'] != "Article":
            continue

        refs = r["CR"].strip().split(";")

        if not len(refs):
            continue


        authors = r['AU'].split(";")
        authors = [x.strip() for x in authors]

        if False:
            for i in range(10):
                print("-"*20)


        uid = r['UT']


        try:
            int(r['PY'])
        except ValueError:
            continue

        r['SO'] = r['SO'].lower() # REMEMBER THIS! lol everything is in lowercase... not case sensitive
        
        if r['SO'].lower() not in journal_keep:
            continue

        yield r
In [56]:
# processes WoS txt output files one by one, counting relevant cooccurrences as it goes
debug=False
dcount=0

files = list(basedir.glob("**/*.txt"))

rows = []

for i in range(100):
    
    f = sample(files,1)[0]
    
    with f.open(encoding='utf8') as pfile:
        r = DictReader(pfile, delimiter="\t")
        my_f_rows = list(r)

    if i % 50 == 0:
        print("File %s/100" % (i,))

    results = list(filter_rows(my_f_rows))
    if not len(results):
        print("zero matches, skipping...")
        continue
        
    r = sample(results,1)[0]
    rows.append({
        "pub": r['PY'], 
        "title": r['TI'].title(),
        "journal": r['SO'].title()
    })
File 0/100
zero matches, skipping...
File 50/100
zero matches, skipping...
zero matches, skipping...
In [57]:
rows = sorted(rows, key=lambda x: -int(x['pub']))
In [58]:
pd.set_option('display.max_rows', 100)
pd.DataFrame.from_records(rows)
Out[58]:
pub title journal
0 2020 Church, State, And "Native Liberty" In The Bel... Comparative Studies In Society And History
1 2019 Dynamics Of Religious Group Growth And Survival Journal For The Scientific Study Of Religion
2 2019 Beyond The Glass Ceiling: Gendering Tourism Ma... Annals Of Tourism Research
3 2019 Siegfried Kracauer'S Differentiating Approach ... Journal Of Historical Sociology
4 2019 The Relationship Between The Position Of Name ... Sociological Methods & Research
5 2019 Pet Loss: Understanding Disenfranchised Grief,... Anthrozoos
6 2019 Using Radical Environmentalist Texts To Uncove... Sociological Methods & Research
7 2019 The Clean Shirt: A Flicker Of Hope In Despair Journal Of Contemporary Ethnography
8 2018 Accumulation Of Disadvantages: Prevalence And ... Social Indicators Research
9 2018 Peers, Parents, And Proactive Criminal Thinkin... Deviant Behavior
10 2017 An Important Milgram-Holocaust Linkage: Formal... Canadian Journal Of Sociology-Cahiers Canadien...
11 2017 Rethinking "Diaspora": Bengal'S Muslims And Hi... Ethnic And Racial Studies
12 2017 Looking Beyond The Church Tax: Families And Th... Journal For The Scientific Study Of Religion
13 2017 Making "Sense" Of Identity: Ethnicity, Nationa... Journal Of Contemporary Ethnography
14 2017 How Obvious Is It? The Content Of Child Sexual... Deviant Behavior
15 2017 Facets Of Agency In Stories Of Transforming Fr... Journal Of Marriage And Family
16 2017 Leisure Spaces, Community, And Third Places Leisure Sciences
17 2017 Bricolage For Self-Sufficiency: An Analysis Of... Sociologia Ruralis
18 2017 Insurgent Rule As Sovereign Mimicry And Mutati... Comparative Studies In Society And History
19 2016 Sociology In The 1980S: The Rise Of Gender (An... Sociology-The Journal Of The British Sociologi...
20 2016 Criteria For Native Food Plant Collection In N... Human Ecology
21 2016 Leadership And Peacemaking: Yitzhak Rabin And ... International Journal Of Intercultural Relations
22 2016 Peter Black, Christopher Stevens, Class And In... Discourse & Society
23 2016 The Effect Of Terrorist Events On Media Portra... Ethnic And Racial Studies
24 2015 Civic Participation Through Volunteerism Among... Sociological Perspectives
25 2015 In The Eye Of The Beholder: The Stratification... Sociological Quarterly
26 2015 Heroin Use Among Adolescents: A Multi-Theoreti... Deviant Behavior
27 2014 Taxation: The Lost History American Journal Of Economics And Sociology
28 2014 Being, Time, And Art: Solzhenitsyn'S Reflectio... Society
29 2014 Reconstructing The Globalisation Of Tourism: A... Annals Of Tourism Research
30 2013 Constraint Negotiation In Serious Leisure A St... Journal Of Leisure Research
31 2013 It Is One Of Those Things That Nobody Can Expl... Journal Of The History Of Sexuality
32 2013 Parental Problem Drinking And Adolescent Psych... Youth & Society
33 2013 Bourdieu In The North: Practical Understanding... Canadian Journal Of Sociology-Cahiers Canadien...
34 2013 Marital Satisfaction, Family Emotional Express... Journal Of Marriage And Family
35 2013 Transition Of Shifting Cultivation And Its Imp... Human Ecology
36 2013 Changing Job Roles In The Norwegian And Uk Fit... Work Employment And Society
37 2013 Doing More With Less: Women'S Leisure During T... Leisure Sciences
38 2012 Application Of The Rural Development Index To ... Social Indicators Research
39 2012 The Night The Raving Died: The Social Construc... Deviant Behavior
40 2011 Parental Leave-Possibility Or Trap? Does Famil... European Sociological Review
41 2011 Metaphor, Social Capital And Sociological Imag... Sociological Review
42 2011 Beyond Texts In Context: Recontextualization A... Discourse & Society
43 2011 Habitus, Work And Contributive Justice Sociology-The Journal Of The British Sociologi...
44 2010 The Reconfiguration Of E-Campaign Practices In... International Sociology
45 2010 The Legacy Of Equality And The Weakness Of Law... European Sociological Review
46 2010 From Renewable To Alternative: Waste Coal, The... Society & Natural Resources
47 2009 Tragic Narratives In Popular Culture: Depictio... Sociological Forum
48 2009 Do Sons Or Daughters Give More Money To Parent... Journal Of Marriage And Family
49 2009 'Strangers In Paradise'? Working-Class Student... Sociology-The Journal Of The British Sociologi...
50 2009 An Assessment Of The Latin Americanization Thesis Ethnic And Racial Studies
51 2009 Getting Drunk And Hooking Up: An Exploratory S... Sociological Spectrum
52 2008 Scientific Discovery In Deep Social Space: Soc... Canadian Journal Of Sociology-Cahiers Canadien...
53 2007 How Does Social Background Affect The Grades A... British Journal Of Sociology Of Education
54 2006 Mollusc Gathering In Northeast Brazil: An Ethn... Human Ecology
55 2006 Early Childhood Cultural Capital, Parental Hab... Poetics
56 2006 Analytic Autoethnography Journal Of Contemporary Ethnography
57 2005 Concurrency And Commitment: Network Scheduling... Journal Of Mathematical Sociology
58 2004 Is Lowest-Low Fertility In Europe Explained By... Population And Development Review
59 2004 Pragmatism, Artificial Intelligence, And Posth... Human Studies
60 2004 The Effects Of Occupational Characteristics On... Work And Occupations
61 2003 Doing Money Work In A Door-To-Door Sales Organ... Symbolic Interaction
62 2001 Testing The Modernization Hypothesis And The S... British Journal Of Sociology
63 2000 Causal Flow Between Human Well-Being And Per C... Social Indicators Research
64 1999 Postmodernism, Institutional Change, And Acade... Social Science Quarterly
65 1996 Islamic Renewal In Niger: From Monolith To Plu... Social Compass
66 1995 Computer-Simulations And Empirical Testing Of ... Sociological Methods & Research
67 1994 Economic-Inequality - New Methods For New Trends American Sociological Review
68 1994 Sampling Weights And Regression-Analysis Sociological Methods & Research
69 1993 Getting Rid Of Troublemakers - High-School Dis... Social Problems
70 1993 Science, Technology, And Congress Society
71 1992 Satisfaction Of What - Post Hoc Versus Real-Ti... Leisure Sciences
72 1990 Capital-Punishment In New-Zealand - An Experim... Deviant Behavior
73 1990 Women And Class - A Problematic Relationship Sociological Review
74 1985 Marriage, Ritual And Profit - The Production O... Social Compass
75 1985 Antecedents Of Pregnancy Among Unmarried Adole... Journal Of Marriage And Family
76 1982 The Structural Factor In Systems Of Communication British Journal Of Sociology
77 1982 Graph Theoretic Blockings K-Plexes And K-Cutpo... Journal Of Mathematical Sociology
78 1982 Existing And Alternative Policy Towards The Ar... Ethnic And Racial Studies
79 1982 A Further Examination Of The Occupational Link... Sociological Quarterly
80 1978 State Of De-Subordination British Journal Of Sociology
81 1978 Significance Of Others In Thought Of Schutz,Al... Sociological Quarterly
82 1977 Roles And Influence Of Political Intellectuals... British Journal Of Sociology
83 1976 Consequences Of Objective And Subjective Statu... Sociological Quarterly
84 1976 Evaluating Competence Of Lawyers Law & Society Review
85 1974 Marxian Epistemology And Transformation Problem Economy And Society
86 1973 Embourgeoisement In Indianapolis Social Problems
87 1968 Patrons, Patronage, And Political Parties Comparative Studies In Society And History
88 1966 Religious Behavior And Prejudice - Some Dimens... Review Of Religious Research
89 1966 Karl Mannheim, Epistemology And Sociology Of K... Sociological Quarterly
90 1965 The Theoretical Perspectives For The Sociologi... Review Of Religious Research
91 1958 Land And Man In History American Journal Of Economics And Sociology
92 1957 Developing Competencies In Democratic Group Le... Sociology Of Education
93 1954 Differential Fertility In The United States 19... American Sociological Review
94 1952 Measuring Family Solidarity American Sociological Review
95 1940 Marriage Rate Among Women Teachers American Sociological Review
96 1929 Present-Day Social Insurance In Germany Social Forces
In [ ]: