stats

# CSIndexbr: Exploring Brazilian Scientific Production in Computer Science

# By Marco Tulio Valente - ASERG/DCC/UFMG
# http://aserg.labsoft.dcc.ufmg.br

# Shell script to update statistics page
# It is called by another script: data/runstats

areas=$(wc -l research-areas-config.csv | awk '{print $1}')

confs=$(wc -l  *-out-confs.csv | awk '/total$/{print $1}')
journals=$(cat *-out-journals.csv | awk -F, '{print $1}' | sort | uniq -c | awk '{print $2}' | wc -l)

confs2=$(cat *-out-confs.csv | awk -F, '$2 > 0 {print $1}' | wc -l)
journals2=$(cat *-out-journals.csv | awk -F, ' $2 > 0 {print $1}' | sort | uniq -c | awk '{print $2}' | wc -l)

papers=$(wc -l  *-out-papers.csv | awk '/total$/{print $1}')

cat *-out-papers.csv > allpapers.csv
papersConf=$(python3 ../utils/getcol.py allpapers.csv 7 | awk '/C/ { print $1 }' | wc -l)
papersJournal=$(python3 ../utils/getcol.py allpapers.csv 7 | awk '/J/ { print $1 }' | wc -l)
paperstop=$(python3 ../utils/getcol.py allpapers.csv 6 | awk '/top/ { print $1 }' | wc -l)
papers_arxiv=$(python3 ../utils/getcol.py allpapers.csv 8 | awk '/arxiv.org/ { print $1 }' | wc -l)


allauthors=$(cat all-researchers.csv | awk -F, '{print $3}' | sort | uniq | wc -l)
authors=$(wc -l profs/all-authors.csv | awk '{print $1}')

papers19=$(cat allpapers.csv | awk -F, '/^2019/ {print $1}' | wc -l)
papers20=$(cat allpapers.csv | awk -F, '/^2020/ {print $1}' | wc -l)
papers21=$(cat allpapers.csv | awk -F, '/^2021/ {print $1}' | wc -l)
papers22=$(cat allpapers.csv | awk -F, '/^2022/ {print $1}' | wc -l)
papers23=$(cat allpapers.csv | awk -F, '/^2023/ {print $1}' | wc -l)
papers24=$(cat allpapers.csv | awk -F, '/^2024/ {print $1}' | wc -l)

top19=$(python3 ../utils/get2col.py allpapers.csv 0 6 | awk '/^2019/ && /top$/ {print $2}' | wc -l)
top20=$(python3 ../utils/get2col.py allpapers.csv 0 6 | awk '/^2020/ && /top$/ {print $2}' | wc -l)
top21=$(python3 ../utils/get2col.py allpapers.csv 0 6 | awk '/^2021/ && /top$/ {print $2}' | wc -l)
top22=$(python3 ../utils/get2col.py allpapers.csv 0 6 | awk '/^2022/ && /top$/ {print $2}' | wc -l)
top23=$(python3 ../utils/get2col.py allpapers.csv 0 6 | awk '/^2023/ && /top$/ {print $2}' | wc -l)
top24=$(python3 ../utils/get2col.py allpapers.csv 0 6 | awk '/^2024/ && /top$/ {print $2}' | wc -l)

#for area in {"se", "pl", "cse", "vision", "or", "cs", "chi", "dbis"}
#do
#  python3 ../utils/get2col.py $area-out-papers.csv 6 7 > temp.csv
#  papers_$area=$(wc -l temp.csv | awk '{print $1}')
#  papers_$area_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
#  papers_$area_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
#  papers_top_$area=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
#  rm temp.csv
#done

python3 ../utils/get2col.py se-out-papers.csv 6 7 > temp.csv
papers_se=$(wc -l temp.csv | awk '{print $1}')
papers_se_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_se_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_se=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py pl-out-papers.csv 6 7 > temp.csv
papers_pl=$(wc -l temp.csv | awk '{print $1}')
papers_pl_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_pl_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_pl=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py cse-out-papers.csv 6 7 > temp.csv
papers_cse=$(wc -l temp.csv | awk '{print $1}')
papers_cse_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_cse_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_cse=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py vision-out-papers.csv 6 7 > temp.csv
papers_vision=$(wc -l temp.csv | awk '{print $1}')
papers_vision_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_vision_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_vision=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py or-out-papers.csv 6 7 > temp.csv
papers_or=$(wc -l temp.csv | awk '{print $1}')
papers_or_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_or_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_or=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py cs-out-papers.csv 6 7 > temp.csv
papers_cs=$(wc -l temp.csv | awk '{print $1}')
papers_cs_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_cs_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_cs=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py chi-out-papers.csv 6 7 > temp.csv
papers_chi=$(wc -l temp.csv | awk '{print $1}')
papers_chi_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_chi_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_chi=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py dbis-out-papers.csv 6 7 > temp.csv
papers_dbis=$(wc -l temp.csv | awk '{print $1}')
papers_dbis_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_dbis_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_dbis=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py arch-out-papers.csv 6 7 > temp.csv
papers_arch=$(wc -l temp.csv | awk '{print $1}')
papers_arch_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_arch_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_arch=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py ds-out-papers.csv 6 7 > temp.csv
papers_ds=$(wc -l temp.csv | awk '{print $1}')
papers_ds_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_ds_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_ds=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py theory-out-papers.csv 6 7 > temp.csv
papers_theory=$(wc -l temp.csv | awk '{print $1}')
papers_theory_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_theory_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_theory=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py ir-out-papers.csv 6 7 > temp.csv
papers_ir=$(wc -l temp.csv | awk '{print $1}')
papers_ir_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_ir_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_ir=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py net-out-papers.csv 6 7 > temp.csv
papers_net=$(wc -l temp.csv | awk '{print $1}')
papers_net_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_net_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_net=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py bio-out-papers.csv 6 7 > temp.csv
papers_bio=$(wc -l temp.csv | awk '{print $1}')
papers_bio_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_bio_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_bio=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py formal-out-papers.csv 6 7 > temp.csv
papers_formal=$(wc -l temp.csv | awk '{print $1}')
papers_formal_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_formal_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_formal=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py robotics-out-papers.csv 6 7 > temp.csv
papers_robotics=$(wc -l temp.csv | awk '{print $1}')
papers_robotics_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_robotics_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_robotics=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py data-out-papers.csv 6 7 > temp.csv
papers_data=$(wc -l temp.csv | awk '{print $1}')
papers_data_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_data_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_data=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py ai-out-papers.csv 6 7 > temp.csv
papers_ai=$(wc -l temp.csv | awk '{print $1}')
papers_ai_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_ai_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_ai=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py security-out-papers.csv 6 7 > temp.csv
papers_security=$(wc -l temp.csv | awk '{print $1}')
papers_security_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_security_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_security=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py hardware-out-papers.csv 6 7 > temp.csv
papers_hardware=$(wc -l temp.csv | awk '{print $1}')
papers_hardware_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_hardware_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_hardware=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

python3 ../utils/get2col.py graphics-out-papers.csv 6 7 > temp.csv
papers_graphics=$(wc -l temp.csv | awk '{print $1}')
papers_graphics_confs=$(cat temp.csv | awk '/C$/ { print $2 }' | wc -l)
papers_graphics_journals=$(cat temp.csv | awk '/J$/ { print $2 }' | wc -l)
papers_top_graphics=$(cat temp.csv | awk '/^top/ { print $1 }' | wc -l)
rm temp.csv

rm allpapers.csv

today=$(date)

echo '<ul>'
echo '<li> Research areas:' $areas
echo '<li> Conferences:' $confs
echo '<li> Conferences with papers by Brazilian professors:' $confs2
echo '<li> Journals:' $journals
echo '<li> Journals with papers by Brazilian professors:' $journals2

echo '<li> Brazilian professors:' $allauthors
echo '<li> Brazilian professors with papers:' $authors '(<a href="profs.html">full list</a>)'

echo '<li> Papers:' $papers
echo '<li> Papers in conferences:' $papersConf
echo '<li> Papers in journals:' $papersJournal
echo '<li> Papers in top venues:' $paperstop
echo '<li> Papers with arXiv preprints:' $papers_arxiv

echo '<li> Papers per year (all papers / papers in top-venues):'
echo '<ul>'
echo '<li>2019:' $papers19 '/' $top19
echo '<li>2020:' $papers20 '/' $top20
echo '<li>2021:' $papers21 '/' $top21
echo '<li>2022:' $papers22 '/' $top22
echo '<li>2023:' $papers23 '/' $top23
echo '<li>2024:' $papers24 '/' $top24
echo '</ul>'

echo '<li> Papers per research area (all papers / papers in confs / papers in journals / papers in top-venues):'
echo '<ul>'
echo '<li>Software Engineering:' $papers_se '/' $papers_se_confs '/' $papers_se_journals '/' $papers_top_se
echo '<li>Programming Languages:' $papers_pl '/' $papers_pl_confs '/' $papers_pl_journals '/' $papers_top_pl
echo '<li>Human-Computer Interaction:' $papers_chi '/' $papers_chi_confs '/' $papers_chi_journals '/' $papers_top_chi
echo '<li>Computer Networks:' $papers_net '/' $papers_net_confs '/' $papers_net_journals '/' $papers_top_net
echo '<li>Distributed Systems:' $papers_ds '/' $papers_ds_confs '/' $papers_ds_journals '/' $papers_top_ds
echo '<li>Computer Architecture & HPC:' $papers_arch '/' $papers_arch_confs '/' $papers_arch_journals '/' $papers_top_arch
echo '<li>Hardware Design:' $papers_hardware '/' $papers_hardware_confs '/' $papers_hardware_journals '/' $papers_top_hardware
echo '<li>Databases & Information Systems:' $papers_dbis '/' $papers_dbis_confs '/' $papers_dbis_journals '/' $papers_top_dbis
echo '<li>Web & Information Retrieval:' $papers_ir '/' $papers_ir_confs '/' $papers_ir_journals '/' $papers_top_ir
echo '<li>Data Mining & Machine Learning:' $papers_data '/' $papers_data_confs '/' $papers_data_journals '/' $papers_top_data
echo '<li>Artificial Intelligence:' $papers_ai '/' $papers_ai_confs '/' $papers_ai_journals '/' $papers_top_ai
echo '<li>Algorithms & Complexity:' $papers_theory '/' $papers_theory_confs '/' $papers_theory_journals '/' $papers_top_theory
echo '<li>Formal Methods & Logic:' $papers_formal '/' $papers_formal_confs '/' $papers_formal_journals '/' $papers_top_formal
echo '<li>Operational Research:' $papers_or '/' $papers_or_confs '/' $papers_or_journals '/' $papers_top_or
echo '<li>Security & Cryptography:' $papers_security '/' $papers_security_confs '/' $papers_security_journals '/' $papers_top_security
echo '<li>Computer Vision:' $papers_vision '/' $papers_vision_confs '/' $papers_vision_journals '/' $papers_top_vision
echo '<li>Computer Graphics:' $papers_graphics '/' $papers_graphics_confs '/' $papers_graphics_journals '/' $papers_top_graphics
echo '<li>Robotics:' $papers_robotics '/' $papers_robotics_confs '/' $papers_robotics_journals '/' $papers_top_robotics
echo '<li>CS Education:' $papers_cse '/' $papers_cse_confs '/' $papers_cse_journals '/' $papers_top_cse
echo '<li>Bioinformatics & Comp. Bio.:' $papers_bio '/' $papers_bio_confs '/' $papers_bio_journals '/' $papers_top_bio
echo '<li>Computer Science (general):' $papers_cs '/' $papers_cs_confs '/' $papers_cs_journals '/' $papers_top_cs
echo '</ul>'

echo '</ul>'

# echo '<li> Papers per department:'
# echo '<ul>'

# cat *-out-papers.csv > allpapers.csv
# python ../utils/getcol.py allpapers.csv 3 | awk '{for(i=1;i<=NF;i++) print $i}' | \
#     sed 's/;//' | sort | uniq -c | sort -r | head -n 15 | awk ' { print "<li>" $2 ": " $1 }'
# rm allpapers.csv

# echo '</ul>'

# echo '<li> Papers in top-conferences per department:'
# echo '<ul>'

# python ../utils/get2col.py allpapers.csv 3 6 | awk ' /top$/ {print $1}'  > allpapers2.csv
# cat allpapers2.csv | awk '{for(i=1;i<=NF;i++) print $i}' | \
#     sed 's/;//' | sort | uniq -c | sort -r | head -n 15 | awk ' { print "<li>" $2 ": " $1 }'
# rm allpapers.csv

# echo '</ul>'

# echo '</ul>'

#up_papers=$(stat -x ../cache/dblp/Marco-Tulio-Valente.xml | awk '/^Change:/ { print $3 " " $4 ", " $6 }')
#up_citations=$(stat -x ../cache/citations/se-citations.csv | awk '/^Change:/ { print $3 " " $4 ", " $6 }')

# echo '<small> <p> Last updated on' $up_papers ' (papers) and ' $up_citations  ' (citations).</p> </small>'

# echo '<small> <p> Last updated on' $up_papers ' (papers).</p> </small>'