{ "cells": [ { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "# First steps on Linux - On Your Own\n", "\n", "The program is :\n", "* using linux commands seen previously and new linux commands\n", "* Counting number of reads, of files\n", "* Plot some basics stats\n", "* Recover sequences of interest from a GFF (capture seq)\n", "* Compare two files (BED)\n", "* Bonus: create a mini workflow with pipe" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# where are you? - `pwd`\n", "\n", "Check where you are in the linux filetree\n", "\n", "" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "slideshow": { "slide_type": "slide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/NOTEBOOK/DAY1-2\n" ] } ], "source": [ "pwd" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# List the content of the directory SV_DATA - `ls`\n", "\n", "### Detailed list of files " ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 16\n", "drwxr-sr-x 2 jovyan users 4096 Jun 17 2022 ASSEMBLIES\n", "drwxr-sr-x 4 jovyan users 4096 Oct 1 18:59 LONG_READS\n", "drwxr-sr-x 3 jovyan users 4096 Oct 2 14:23 REF\n", "drwxr-sr-x 2 jovyan users 4096 Oct 1 19:54 SHORT_READS\n" ] } ], "source": [ "ls -l ../../SV_DATA" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### With -R, --recursive list subdirectories recursively" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "../../SV_DATA:\n", "ASSEMBLIES LONG_READS REF SHORT_READS\n", "\n", "../../SV_DATA/ASSEMBLIES:\n", "5417_assembly.fasta A8_assembly.fasta assembly-stats.txt\n", "\n", "../../SV_DATA/LONG_READS:\n", "5417 A8\n", "\n", "../../SV_DATA/LONG_READS/5417:\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_0_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_10_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_1_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_11_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_12_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_13_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_14_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_2_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_3_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_4_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_5_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_6_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_7_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_8_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_9_0.fastq.gz\n", "\n", "../../SV_DATA/LONG_READS/A8:\n", "fastq_runid_023874091309670a42f7a7c51d363b94fa86dd50_0_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_10_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_1_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_11_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_2_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_3_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_4_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_5_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_6_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_7_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_8_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_9_0.fastq.gz\n", "\n", "../../SV_DATA/REF:\n", "GCA_002220235.1_ASM222023v1_genomic.fna\n", "GCA_002220235.1_ASM222023v1_genomic.fna.0123\n", "GCA_002220235.1_ASM222023v1_genomic.fna.amb\n", "GCA_002220235.1_ASM222023v1_genomic.fna.ann\n", "GCA_002220235.1_ASM222023v1_genomic.fna.bwt.2bit.64\n", "GCA_002220235.1_ASM222023v1_genomic.fna.fai\n", "GCA_002220235.1_ASM222023v1_genomic.fna.pac\n", "GCA_002220235.1_ASM222023v1_genomic.gff\n", "qtl.bed\n", "\n", "../../SV_DATA/SHORT_READS:\n", "1613_R1.fastq.gz 685_R2.fastq.gz B218_R1.fastq.gz D119_R2.fastq.gz\n", "1613_R2.fastq.gz 716_R1.fastq.gz B218_R2.fastq.gz E2_R1.fastq.gz\n", "1868_R1.fastq.gz 716_R2.fastq.gz B8_R1.fastq.gz E2_R2.fastq.gz\n", "1868_R2.fastq.gz A1_R1.fastq.gz B8_R2.fastq.gz E318_R1.fastq.gz\n", "4752_R1.fastq.gz A1_R2.fastq.gz C218_R1.fastq.gz E318_R2.fastq.gz\n", "4752_R2.fastq.gz A8_R1.fastq.gz C218_R2.fastq.gz G11_R1.fastq.gz\n", "5417_R1.fastq.gz A8_R2.fastq.gz C2_R1.fastq.gz G11_R2.fastq.gz\n", "5417_R2.fastq.gz B1_R1.fastq.gz C2_R2.fastq.gz H44_R1.fastq.gz\n", "685_R1.fastq.gz B1_R2.fastq.gz D119_R1.fastq.gz H44_R2.fastq.gz\n" ] } ], "source": [ "ls -R ../../SV_DATA" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# List the content of the directory SHORT_READS - `ls -l`" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 7067920\n", "-rwxr-x--- 1 jovyan users 115855630 Oct 1 18:24 1613_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 129668496 Oct 1 18:24 1613_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 108366349 Oct 1 18:24 1868_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 120549974 Oct 1 18:24 1868_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 106516571 Oct 1 18:24 4752_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 118981710 Oct 1 18:24 4752_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 123634824 Oct 1 18:24 5417_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 149650318 Oct 1 18:24 5417_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 113973867 Oct 1 18:24 685_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 127925521 Oct 1 18:24 685_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 108607336 Oct 1 18:24 716_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 121545822 Oct 1 18:24 716_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 399056884 Oct 1 18:24 A1_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 423279636 Oct 1 18:24 A1_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 126125733 Oct 1 18:24 A8_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 152151244 Oct 1 18:24 A8_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 126130397 Oct 1 18:24 B1_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 151606359 Oct 1 18:24 B1_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 313591987 Oct 1 18:24 B218_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 326574109 Oct 1 18:24 B218_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 131613912 Oct 1 18:24 B8_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 157774699 Oct 1 18:24 B8_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 391619599 Oct 1 18:24 C218_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 408700056 Oct 1 18:24 C218_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 123634531 Oct 1 18:25 C2_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 149384144 Oct 1 18:25 C2_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 261162790 Oct 1 18:25 D119_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 274437605 Oct 1 18:25 D119_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 124037165 Oct 1 18:25 E2_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 149177136 Oct 1 18:25 E2_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 338337601 Oct 1 18:25 E318_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 355161556 Oct 1 18:25 E318_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 122978817 Oct 1 18:25 G11_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 147336917 Oct 1 18:25 G11_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 314797219 Oct 1 18:25 H44_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 323490585 Oct 1 18:25 H44_R2.fastq.gz\n" ] } ], "source": [ "ls -l ../../SV_DATA/SHORT_READS" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 6.8G\n", "-rwxr-x--- 1 jovyan users 111M Oct 1 18:24 1613_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 124M Oct 1 18:24 1613_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 104M Oct 1 18:24 1868_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 115M Oct 1 18:24 1868_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 102M Oct 1 18:24 4752_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 114M Oct 1 18:24 4752_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 118M Oct 1 18:24 5417_R1.fastq.gz\n", "-rwxr-x--- 1 jovyan users 143M Oct 1 18:24 5417_R2.fastq.gz\n", "-rwxr-x--- 1 jovyan users 109M Oct 1 18:24 685_R1.fastq.gz\n" ] } ], "source": [ "# with -h, --human-readable and only print the 10th lines of the list files\n", "ls -lh ../../SV_DATA/SHORT_READS | head" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----------------------------------------------------\n", "\n", "__EXERCICES:__\n", "\n", "* 1 - List the content of the directory REF by using its absolute path in first then its relative path - `ls` command\n", "* 2 - List the content of the directory LONG_READS with the `ls` command and the option `-R`\n", "* 3 - List the content of the directory ifbdata/public/swissprot with the `ls` command and the option `-al` or `-a -l`\n", "\n", "----------------------------------------------------\n", "\n", "__1 - List the content of the directory REF by using its absolute path in first then its relative path - `ls` command__\n", "\n", "> -t sort by time, newest first; \n", "\n", "> -r, --reverse reverse order while sorting" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 101648\n", "-rw-r--r-- 1 jovyan users 65 Oct 2 14:17 qtl.bed\n", "-rw-r--r-- 1 jovyan users 681 Sep 25 19:37 GCA_002220235.1_ASM222023v1_genomic.fna.fai\n", "-rw-r--r-- 1 jovyan users 48991633 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.bwt.2bit.64\n", "-rw-r--r-- 1 jovyan users 30148640 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.0123\n", "-rw-r--r-- 1 jovyan users 308 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.amb\n", "-rw-r--r-- 1 jovyan users 1587 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.ann\n", "-rw-r--r-- 1 jovyan users 3768582 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.pac\n", "-rw------- 1 jovyan users 5886829 Sep 23 15:25 GCA_002220235.1_ASM222023v1_genomic.gff\n", "-rw------- 1 jovyan users 15263945 Sep 23 01:29 GCA_002220235.1_ASM222023v1_genomic.fna\n" ] } ], "source": [ "#1\n", "ls -lrt /home/jovyan/SV_DATA/REF" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----------------------------------------------------\n", "\n", "__2 - List the content of the directory LONG_READS with the `ls` command and the option `-R`__\n" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "../../SV_DATA/LONG_READS:\n", "5417 A8\n", "\n", "../../SV_DATA/LONG_READS/5417:\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_0_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_10_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_1_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_11_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_12_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_13_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_14_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_2_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_3_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_4_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_5_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_6_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_7_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_8_0.fastq.gz\n", "fastq_runid_d9b45d7d98e41184ec96f40f59021521f703067d_9_0.fastq.gz\n", "\n", "../../SV_DATA/LONG_READS/A8:\n", "fastq_runid_023874091309670a42f7a7c51d363b94fa86dd50_0_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_10_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_1_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_11_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_2_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_3_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_4_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_5_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_6_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_7_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_8_0.fastq.gz\n", "fastq_runid_243e56cb3c6676a3d235987c3ae536c7646d460a_9_0.fastq.gz\n" ] } ], "source": [ "ls -R ../../SV_DATA/LONG_READS" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "-----------------------------------------------------------------\n", "\n", "__3 - List the content of the directory ifbdata/public/swissprot with the `ls` command and the option `-al` or `-a -l`__\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "ls ~/ifbdata/public/" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Usage: ls [OPTION]... [FILE]...\n", "List information about the FILEs (the current directory by default).\n", "Sort entries alphabetically if none of -cftuvSUX nor --sort is specified.\n", "\n", "Mandatory arguments to long options are mandatory for short options too.\n", " -a, --all do not ignore entries starting with .\n", " -A, --almost-all do not list implied . and ..\n", " --author with -l, print the author of each file\n", " -b, --escape print C-style escapes for nongraphic characters\n", " --block-size=SIZE with -l, scale sizes by SIZE when printing them;\n", " e.g., '--block-size=M'; see SIZE format below\n", " -B, --ignore-backups do not list implied entries ending with ~\n", " -c with -lt: sort by, and show, ctime (time of last\n", " modification of file status information);\n", " with -l: show ctime and sort by name;\n", " otherwise: sort by ctime, newest first\n", " -C list entries by columns\n", " --color[=WHEN] colorize the output; WHEN can be 'always' (default\n", " if omitted), 'auto', or 'never'; more info below\n", " -d, --directory list directories themselves, not their contents\n", " -D, --dired generate output designed for Emacs' dired mode\n", " -f do not sort, enable -aU, disable -ls --color\n", " -F, --classify append indicator (one of */=>@|) to entries\n", " --file-type likewise, except do not append '*'\n", " --format=WORD across -x, commas -m, horizontal -x, long -l,\n", " single-column -1, verbose -l, vertical -C\n", " --full-time like -l --time-style=full-iso\n", " -g like -l, but do not list owner\n", " --group-directories-first\n", " group directories before files;\n", " can be augmented with a --sort option, but any\n", " use of --sort=none (-U) disables grouping\n", " -G, --no-group in a long listing, don't print group names\n", " -h, --human-readable with -l and -s, print sizes like 1K 234M 2G etc.\n", " --si likewise, but use powers of 1000 not 1024\n", " -H, --dereference-command-line\n", " follow symbolic links listed on the command line\n", " --dereference-command-line-symlink-to-dir\n", " follow each command line symbolic link\n", " that points to a directory\n", " --hide=PATTERN do not list implied entries matching shell PATTERN\n", " (overridden by -a or -A)\n", " --hyperlink[=WHEN] hyperlink file names; WHEN can be 'always'\n", " (default if omitted), 'auto', or 'never'\n", " --indicator-style=WORD append indicator with style WORD to entry names:\n", " none (default), slash (-p),\n", " file-type (--file-type), classify (-F)\n", " -i, --inode print the index number of each file\n", " -I, --ignore=PATTERN do not list implied entries matching shell PATTERN\n", " -k, --kibibytes default to 1024-byte blocks for disk usage;\n", " used only with -s and per directory totals\n", " -l use a long listing format\n", " -L, --dereference when showing file information for a symbolic\n", " link, show information for the file the link\n", " references rather than for the link itself\n", " -m fill width with a comma separated list of entries\n", " -n, --numeric-uid-gid like -l, but list numeric user and group IDs\n", " -N, --literal print entry names without quoting\n", " -o like -l, but do not list group information\n", " -p, --indicator-style=slash\n", " append / indicator to directories\n", " -q, --hide-control-chars print ? instead of nongraphic characters\n", " --show-control-chars show nongraphic characters as-is (the default,\n", " unless program is 'ls' and output is a terminal)\n", " -Q, --quote-name enclose entry names in double quotes\n", " --quoting-style=WORD use quoting style WORD for entry names:\n", " literal, locale, shell, shell-always,\n", " shell-escape, shell-escape-always, c, escape\n", " (overrides QUOTING_STYLE environment variable)\n", " -r, --reverse reverse order while sorting\n", " -R, --recursive list subdirectories recursively\n", " -s, --size print the allocated size of each file, in blocks\n", " -S sort by file size, largest first\n", " --sort=WORD sort by WORD instead of name: none (-U), size (-S),\n", " time (-t), version (-v), extension (-X)\n", " --time=WORD change the default of using modification times;\n", " access time (-u): atime, access, use;\n", " change time (-c): ctime, status;\n", " birth time: birth, creation;\n", " with -l, WORD determines which time to show;\n", " with --sort=time, sort by WORD (newest first)\n", " --time-style=TIME_STYLE time/date format with -l; see TIME_STYLE below\n", " -t sort by time, newest first; see --time\n", " -T, --tabsize=COLS assume tab stops at each COLS instead of 8\n", " -u with -lt: sort by, and show, access time;\n", " with -l: show access time and sort by name;\n", " otherwise: sort by access time, newest first\n", " -U do not sort; list entries in directory order\n", " -v natural sort of (version) numbers within text\n", " -w, --width=COLS set output width to COLS. 0 means no limit\n", " -x list entries by lines instead of by columns\n", " -X sort alphabetically by entry extension\n", " -Z, --context print any security context of each file\n", " -1 list one file per line. Avoid '\\n' with -q or -b\n", " --help display this help and exit\n", " --version output version information and exit\n", "\n", "The SIZE argument is an integer and optional unit (example: 10K is 10*1024).\n", "Units are K,M,G,T,P,E,Z,Y (powers of 1024) or KB,MB,... (powers of 1000).\n", "Binary prefixes can be used, too: KiB=K, MiB=M, and so on.\n", "\n", "The TIME_STYLE argument can be full-iso, long-iso, iso, locale, or +FORMAT.\n", "FORMAT is interpreted like in date(1). If FORMAT is FORMAT1FORMAT2,\n", "then FORMAT1 applies to non-recent files and FORMAT2 to recent files.\n", "TIME_STYLE prefixed with 'posix-' takes effect only outside the POSIX locale.\n", "Also the TIME_STYLE environment variable sets the default style to use.\n", "\n", "Using color to distinguish file types is disabled both by default and\n", "with --color=never. With --color=auto, ls emits color codes only when\n", "standard output is connected to a terminal. The LS_COLORS environment\n", "variable can change the settings. Use the dircolors command to set it.\n", "\n", "Exit status:\n", " 0 if OK,\n", " 1 if minor problems (e.g., cannot access subdirectory),\n", " 2 if serious trouble (e.g., cannot access command-line argument).\n", "\n", "GNU coreutils online help: \n", "Full documentation \n", "or available locally via: info '(coreutils) ls invocation'\n" ] } ], "source": [ "ls --help" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Working with the wildcard `*`\n", "\n", "\n", "#### What is a wildcard ?\n", "\n", "* It is a character that can be used as a substitute for any of a class of characters in a search\n", "\n", "* Can be used with numerous linux commands \n", "\n", "#### How to use the wildcard `*`\n", "\n", "* \\* can represent zero characters, all single characters or any string\n", "\n", "###### Example\n", "`ls *fastq` will display __only the files ended by fastq__ in the directory where I am working1\n", "\n", "#### Try the following commands using (or not) the wildcad *" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "GCA_002220235.1_ASM222023v1_genomic.fna\n", "GCA_002220235.1_ASM222023v1_genomic.fna.0123\n", "GCA_002220235.1_ASM222023v1_genomic.fna.amb\n", "GCA_002220235.1_ASM222023v1_genomic.fna.ann\n", "GCA_002220235.1_ASM222023v1_genomic.fna.bwt.2bit.64\n", "GCA_002220235.1_ASM222023v1_genomic.fna.fai\n", "GCA_002220235.1_ASM222023v1_genomic.fna.pac\n", "GCA_002220235.1_ASM222023v1_genomic.gff\n", "qtl.bed\n" ] } ], "source": [ "ls /home/jovyan/SV_DATA/REF" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/SV_DATA/REF/GCA_002220235.1_ASM222023v1_genomic.fna\n" ] } ], "source": [ "ls /home/jovyan/SV_DATA/REF/*fna" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "__EXERCICES__\n", "\n", "* 4 - List the content of the directory SHORT_READS. - `ls command`\n", "* 5 - List only the files starting by B (in the directory SHORT_READS ) - `ls command & *`\n", "* 6 - List only the files with the word R1 (in the directory SHORT_READS ) - `ls command & *`\n", "\n", "---------------------\n", "\n", "__4 - List the content of the directory SHORT_READS. - `ls command`__" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1613_R1.fastq.gz 685_R2.fastq.gz B218_R1.fastq.gz D119_R2.fastq.gz\n", "1613_R2.fastq.gz 716_R1.fastq.gz B218_R2.fastq.gz E2_R1.fastq.gz\n", "1868_R1.fastq.gz 716_R2.fastq.gz B8_R1.fastq.gz E2_R2.fastq.gz\n", "1868_R2.fastq.gz A1_R1.fastq.gz B8_R2.fastq.gz E318_R1.fastq.gz\n", "4752_R1.fastq.gz A1_R2.fastq.gz C218_R1.fastq.gz E318_R2.fastq.gz\n", "4752_R2.fastq.gz A8_R1.fastq.gz C218_R2.fastq.gz G11_R1.fastq.gz\n", "5417_R1.fastq.gz A8_R2.fastq.gz C2_R1.fastq.gz G11_R2.fastq.gz\n", "5417_R2.fastq.gz B1_R1.fastq.gz C2_R2.fastq.gz H44_R1.fastq.gz\n", "685_R1.fastq.gz B1_R2.fastq.gz D119_R1.fastq.gz H44_R2.fastq.gz\n" ] } ], "source": [ "ls ~/SV_DATA/SHORT_READS" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "--------------\n", "\n", "__5 - List only the files starting by B (in the directory SHORT_READS ) - `ls command & *`__" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/SV_DATA/SHORT_READS/B1_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B1_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B218_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B218_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B8_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B8_R2.fastq.gz\n" ] } ], "source": [ "ls ~/SV_DATA/SHORT_READS/B*" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "_starting by B or C_" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/SV_DATA/SHORT_READS/B1_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B1_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B218_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B218_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B8_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B8_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C218_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C218_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C2_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C2_R2.fastq.gz\n" ] } ], "source": [ "ls ~/SV_DATA/SHORT_READS/[BC]*" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "_starting by a letter between B and E_" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/SV_DATA/SHORT_READS/B1_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B1_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B218_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B218_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B8_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B8_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C218_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C218_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C2_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C2_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/D119_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/D119_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/E2_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/E2_R2.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/E318_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/E318_R2.fastq.gz\n" ] } ], "source": [ "ls ~/SV_DATA/SHORT_READS/[B-E]*" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "---------\n", "__6 -List only the files with the word R1 (in the directory SHORT_READS ) - `ls command & *`__" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/SV_DATA/SHORT_READS/1613_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/1868_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/4752_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/5417_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/685_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/716_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/A1_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/A8_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B1_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B218_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/B8_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C218_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/C2_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/D119_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/E2_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/E318_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/G11_R1.fastq.gz\n", "/home/jovyan/SV_DATA/SHORT_READS/H44_R1.fastq.gz\n" ] } ], "source": [ "ls ~/SV_DATA/SHORT_READS/*R1*" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "##### Check that we have the same number of fastq files \"Reverse\" (R1) and \"Forward\" (R2)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "18\n" ] } ], "source": [ "ls ~/SV_DATA/SHORT_READS/*R1* | wc -l" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "18\n" ] } ], "source": [ "ls ~/SV_DATA/SHORT_READS/*R2* | wc -l" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "##### Display the begiing of the fastq file" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "@M02713:279:GW200319000:1:2106:14798:1364 1:N:0:GGATCTGA+GGTAAGCT\n", "AACAGTCGCAAAGATTCTCTCAGAGAATGAGAAGACTTCCTTTTCTCGTCGTCATCGTTCTTCTTCACGCCATTTCCACCTCCACCACCACCACCACCTTGTGCCATGCTGCTGCTGCTGCTGATGAAACGATGATCGTCTCTCTGACTCGCTCTGACTCTCTTACTGTGAGGAAACAAACGACAAGTTTTTTTTCCTTCTTTTCCTCATCAGACGACGAGAGAGTTTCTCGCAAGTCAAAATCACCTT\n", "+\n", "AAAAAFFADDADGBG1F3F3BGGGGHHBFFFHBGFFGFFFHHH1GHHEGA/EEHFBG/FEFHGFHB2GHHGGGHFHFGFGHFGFHHH/FFAEAGFFHFFHGEFHHHHGHHFH111FB>GCHGFHGHHFHHG/CCFHHEGHEFFHHHHHFFFCFHHFFGHFHFHHHHHH\n", "\n", "#### EXERCICES\n", "\n", "* 7 - Go to the directory _REF_ and check in the prompt you have changed correctly your working directory `pwd`\n", "* 8 - List the dir content with `ls`\n" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Where am I?\n", "/home/jovyan/SV_DATA/REF\n", "\n", "Where am I?\n", "/home/jovyan/SV_DATA/REF\n" ] } ], "source": [ "#7\n", "echo \"Where am I?\"\n", "pwd \n", "\n", "cd /home/jovyan/SV_DATA/REF\n", "\n", "echo -e \"\\nWhere am I?\"\n", "pwd \n" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 101648\n", "-rw------- 1 jovyan users 15263945 Sep 23 01:29 GCA_002220235.1_ASM222023v1_genomic.fna\n", "-rw------- 1 jovyan users 5886829 Sep 23 15:25 GCA_002220235.1_ASM222023v1_genomic.gff\n", "-rw-r--r-- 1 jovyan users 3768582 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.pac\n", "-rw-r--r-- 1 jovyan users 1587 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.ann\n", "-rw-r--r-- 1 jovyan users 308 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.amb\n", "-rw-r--r-- 1 jovyan users 30148640 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.0123\n", "-rw-r--r-- 1 jovyan users 48991633 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.bwt.2bit.64\n", "-rw-r--r-- 1 jovyan users 681 Sep 25 19:37 GCA_002220235.1_ASM222023v1_genomic.fna.fai\n", "-rw-r--r-- 1 jovyan users 65 Oct 2 14:17 qtl.bed\n" ] } ], "source": [ "ls -lrt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ " #### Let's use the command `cd ..`\n", "\n", "* `cd ..` : go one folder up\n", "* `cd ../..` : go two folders up\n", "\n", "-----------------------------\n", "\n", "#### EXERCICES\n", "\n", "* 9 - Go to the _LONG_READ/A8_ directory using `../`\n", "* 10 - Go to the _SHORT_READS_. From this directory, and without any change in your working dir, list what’s in _REF directory_\n", "* 11 - List the _ifbdata/public/_ directory using `-R option`. What is there in this dir ?\n", "* 12 - Come back to your home directory using cd\n", "\n", "-------------------------------" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/SV_DATA/LONG_READS/A8\n" ] } ], "source": [ "#9\n", "cd ../LONG_READS/A8/\n", "pwd" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/SV_DATA/SHORT_READS\n" ] } ], "source": [ "#10\n", "cd ../../SHORT_READS\n", "pwd" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "GCA_002220235.1_ASM222023v1_genomic.fna\n", "GCA_002220235.1_ASM222023v1_genomic.fna.0123\n", "GCA_002220235.1_ASM222023v1_genomic.fna.amb\n", "GCA_002220235.1_ASM222023v1_genomic.fna.ann\n", "GCA_002220235.1_ASM222023v1_genomic.fna.bwt.2bit.64\n", "GCA_002220235.1_ASM222023v1_genomic.fna.fai\n", "GCA_002220235.1_ASM222023v1_genomic.fna.pac\n", "GCA_002220235.1_ASM222023v1_genomic.gff\n", "qtl.bed\n" ] } ], "source": [ "ls ../REF" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Where am I?\n", "/home/jovyan/SV_DATA/SHORT_READS\n", "\n", "Where am I?\n", "/home/jovyan\n" ] } ], "source": [ "#12\n", "#7\n", "echo \"Where am I?\"\n", "pwd \n", "\n", "cd \n", "\n", "echo -e \"\\nWhere am I?\"\n", "pwd " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Some Commands\n", "\n", "\n", "| | |\n", "|:------|:------|\n", "| pwd | Display the full path of the current directory|\n", "| ls | List all files/directories|\n", "| cd DIR_NAME| Change the working directory|\n", "| rm nom_fichier | Remove a directory\n", "| cp file1 file2 | Make a copy of file1 and calls it file2 |\n", "| cp FILE_NAME DIR_NAME | Copy the file FILE_NAME in the directory DIR_NAME |\n", "| cp FILE DIR/NEW_FILE | Mix of 2 |\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Manipulating Files and Folders\n", "\n", "#### Exercices\n", "\n", "```{Note}\n", "Be careful with the `rm` command, in linux, there is no bin !!!\n", "```\n", "\n", "* 13 - Remove the file SV-DATA.tar.gz within your home directory - `rm`\n", "* 14 - Copy the files Day1_Get_Data.ipynb and myFirstJupyterbook.ipŷnb into the directory NOTEBOOK\n", "* 15 - Remove the files Day1_Get_Data.ipynb and myFirstJupyterbook.ipynb in your home directory - `rm command`\n", "* 16 - Create a directory called SCRIPT in your home directory - `mkdir`\n", "\n", "-----------------------------------------" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 7403360\n", "-rw-r--r-- 1 jovyan users 9447 Oct 2 13:35 Day1_Get_Data.ipynb\n", "drwxr-xr-x 4 root root 4096 Sep 29 06:35 ifbdata\n", "drwxr-sr-x 3 jovyan users 4096 Oct 2 13:20 NOTEBOOK\n", "drwxr-sr-x 6 jovyan users 4096 Oct 1 19:00 SV_DATA\n", "-rw-r--r-- 1 jovyan users 7581002472 Oct 1 21:10 SV_DATA.tar.gz\n", "-rw-r--r-- 1 jovyan users 1425 Oct 2 13:22 Untitled.ipynb\n", "drwxr-xr-x 2 jovyan 1000 4096 Sep 29 06:35 work\n" ] } ], "source": [ "ls /home/jovyan -l" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "rm: cannot remove '/home/jovyan/SV_DATA.tar.gz': No such file or directory\n", "total 32\n", "-rw-r--r-- 1 jovyan users 9447 Oct 2 13:35 Day1_Get_Data.ipynb\n", "drwxr-xr-x 4 root root 4096 Sep 29 06:35 ifbdata\n", "drwxr-sr-x 3 jovyan users 4096 Oct 2 13:20 NOTEBOOK\n", "drwxr-sr-x 6 jovyan users 4096 Oct 1 19:00 SV_DATA\n", "-rw-r--r-- 1 jovyan users 1425 Oct 2 13:22 Untitled.ipynb\n", "drwxr-xr-x 2 jovyan 1000 4096 Sep 29 06:35 work\n" ] } ], "source": [ "rm /home/jovyan/SV_DATA.tar.gz\n", "ls /home/jovyan/ -l" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "
\n", "\n", "# Let's practice some bioinformatics tools" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### TIPS\n", "\n", "For saving the output of a command to a file, we will use the `>` character \n", "\n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "
\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 1 - Manipulation of `gff` and `bed` files \n", "\n", "* Recover sequences of interest from a GFF (capture seq)\n", "* Compare two files (BED)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Download the file that contains the rice genome annotation (gff format) and the sequence of the 12 chromosomes (fasta format) - `wget`\n", "* the url of the fasta file (all.chr.con):\n", "http://rice.uga.edu/pub/data/Eukaryotic_Projects/o_sativa/annotation_dbs/pseudomolecules/version_7.0/all.dir/all.chrs.con\n", "\n", "\n", "* the url of the gff file (all.gff3) : http://rice.uga.edu/pub/data/Eukaryotic_Projects/o_sativa/annotation_dbs/pseudomolecules/version_7.0/all.dir/all.gff3\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### EXERCICE : \n", "\n", "* 1 - Go to the bank directory and type the correct wget command\n", "* 2 - Download the gff file (all.gff3) and the fasta file (all.chrs.con) in the bank directory \n", "* 3 - List the content of the bank directory\n", "* 4 - Count the line number of the gff file\n", "* 5 - Displays the firsts and lasts lines of the gff file\n", "\n" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/NOTEBOOK/DAY1-2\n", "total 101644\n", "-rw------- 1 jovyan users 15263945 Sep 23 01:29 GCA_002220235.1_ASM222023v1_genomic.fna\n", "-rw------- 1 jovyan users 5886829 Sep 23 15:25 GCA_002220235.1_ASM222023v1_genomic.gff\n", "-rw-r--r-- 1 jovyan users 3768582 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.pac\n", "-rw-r--r-- 1 jovyan users 1587 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.ann\n", "-rw-r--r-- 1 jovyan users 308 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.amb\n", "-rw-r--r-- 1 jovyan users 30148640 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.0123\n", "-rw-r--r-- 1 jovyan users 48991633 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.bwt.2bit.64\n", "-rw-r--r-- 1 jovyan users 681 Sep 25 19:37 GCA_002220235.1_ASM222023v1_genomic.fna.fai\n" ] } ], "source": [ "#1\n", "pwd\n", "cd ../../SV_DATA/REF\n", "ls -lrt\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2023-10-03 12:59:54-- http://rice.uga.edu/pub/data/Eukaryotic_Projects/o_sativa/annotation_dbs/pseudomolecules/version_7.0/all.dir/all.chrs.con\n", "Resolving rice.uga.edu (rice.uga.edu)... 128.192.162.131\n", "Connecting to rice.uga.edu (rice.uga.edu)|128.192.162.131|:80... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 380710511 (363M)\n", "Saving to: ‘all.chrs.con’\n", "\n", "all.chrs.con 100%[===================>] 363.07M 5.69MB/s in 3m 35s \n", "\n", "2023-10-03 13:03:29 (1.69 MB/s) - ‘all.chrs.con’ saved [380710511/380710511]\n", "\n", "--2023-10-03 13:03:29-- http://rice.uga.edu/pub/data/Eukaryotic_Projects/o_sativa/annotation_dbs/pseudomolecules/version_7.0/all.dir/all.gff3\n", "Resolving rice.uga.edu (rice.uga.edu)... 128.192.162.131\n", "Connecting to rice.uga.edu (rice.uga.edu)|128.192.162.131|:80... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 81498659 (78M)\n", "Saving to: ‘all.gff3’\n", "\n", "all.gff3 100%[===================>] 77.72M 2.78MB/s in 25s \n", "\n", "2023-10-03 13:03:55 (3.07 MB/s) - ‘all.gff3’ saved [81498659/81498659]\n", "\n" ] } ], "source": [ "#2\n", "wget http://rice.uga.edu/pub/data/Eukaryotic_Projects/o_sativa/annotation_dbs/pseudomolecules/version_7.0/all.dir/all.chrs.con\n", "wget http://rice.uga.edu/pub/data/Eukaryotic_Projects/o_sativa/annotation_dbs/pseudomolecules/version_7.0/all.dir/all.gff3" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 553024\n", "-rw-r--r-- 1 jovyan users 681 Sep 25 19:37 GCA_002220235.1_ASM222023v1_genomic.fna.fai\n", "-rw-r--r-- 1 jovyan users 48991633 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.bwt.2bit.64\n", "-rw-r--r-- 1 jovyan users 30148640 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.0123\n", "-rw-r--r-- 1 jovyan users 308 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.amb\n", "-rw-r--r-- 1 jovyan users 1587 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.ann\n", "-rw-r--r-- 1 jovyan users 3768582 Sep 24 10:45 GCA_002220235.1_ASM222023v1_genomic.fna.pac\n", "-rw------- 1 jovyan users 5886829 Sep 23 15:25 GCA_002220235.1_ASM222023v1_genomic.gff\n", "-rw------- 1 jovyan users 15263945 Sep 23 01:29 GCA_002220235.1_ASM222023v1_genomic.fna\n", "-rw-r--r-- 1 jovyan users 81498659 Feb 7 2012 all.gff3\n", "-rw-r--r-- 1 jovyan users 380710511 Oct 31 2011 all.chrs.con\n" ] } ], "source": [ "ls -lt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### More information about gff format\n", "\n", "* the gff (General Feature Format) file is a simple tab-delimited text file for describing genomic features \n", "\n", "* see here to understand the gff format : https://www.ensembl.org/info/website/upload/gff.html" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### Counts the number of gene (line with the word \"gene\", column 3)\n", "\n", "* We will use a new linux command `grep` \n", "* for example, we want to count the number of sequences in the fasta file just downloaded " ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ ">Chr1\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n" ] } ], "source": [ "head -n 20 all.chrs.con" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\n", "NNNNNN\n" ] } ], "source": [ "tail -n 20 all.chrs.con" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "7464929 all.chrs.con\n" ] } ], "source": [ "wc -l all.chrs.con" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### Search and count for > motif in the rice genome (fasta)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr1\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr2\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr3\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr4\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr5\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr6\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr7\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr8\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr9\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr10\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr11\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr12\n" ] } ], "source": [ "grep \">\" all.chrs.con" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "12\n" ] } ], "source": [ "grep \">\" all.chrs.con -c" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### Search for the keyword `exon` in the gff file" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Chr1\tMSU_osa1r7\texon\t2903\t3268\t.\t+\t.\tID=LOC_Os01g01010.1:exon_1;Parent=LOC_Os01g01010.1\n", "Chr1\tMSU_osa1r7\texon\t3354\t3616\t.\t+\t.\tID=LOC_Os01g01010.1:exon_2;Parent=LOC_Os01g01010.1\n", "Chr1\tMSU_osa1r7\texon\t4357\t4455\t.\t+\t.\tID=LOC_Os01g01010.1:exon_3;Parent=LOC_Os01g01010.1\n", "Chr1\tMSU_osa1r7\texon\t5457\t5560\t.\t+\t.\tID=LOC_Os01g01010.1:exon_4;Parent=LOC_Os01g01010.1\n", "Chr1\tMSU_osa1r7\texon\t7136\t7944\t.\t+\t.\tID=LOC_Os01g01010.1:exon_5;Parent=LOC_Os01g01010.1\n", "Chr1\tMSU_osa1r7\texon\t8028\t8150\t.\t+\t.\tID=LOC_Os01g01010.1:exon_6;Parent=LOC_Os01g01010.1\n", "Chr1\tMSU_osa1r7\texon\t8232\t8320\t.\t+\t.\tID=LOC_Os01g01010.1:exon_7;Parent=LOC_Os01g01010.1\n", "Chr1\tMSU_osa1r7\texon\t8408\t8608\t.\t+\t.\tID=LOC_Os01g01010.1:exon_8;Parent=LOC_Os01g01010.1\n", "Chr1\tMSU_osa1r7\texon\t9210\t9617\t.\t+\t.\tID=LOC_Os01g01010.1:exon_9;Parent=LOC_Os01g01010.1\n", "Chr1\tMSU_osa1r7\texon\t10104\t10187\t.\t+\t.\tID=LOC_Os01g01010.1:exon_10;Parent=LOC_Os01g01010.1\n", "grep: write error: Broken pipe\n" ] } ], "source": [ "grep \"exon\" all.gff3 | head" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "312567\n" ] } ], "source": [ "grep \"exon\" all.gff3 -c" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "312495\n" ] } ], "source": [ "grep -e \"\\sexon\\s\" all.gff3 -c" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### EXERCICE" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* 1 - search/select only the lines with the gene keyword int the gff file - `grep`\n", "* 2 - save in a new file (all_gene.gff3) all the lines with the gene keyword in the gff file all.gff3\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### Compare two bed files\n", "\n", "We have found 3 genomic regions of interest from a previous analysis in our lab. The file qtl.bed contains informations about theses regions (chromosome, start, stop) / bed format " ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Chr1\tMSU_osa1r7\tgene\t2903\t10817\t.\t+\t.\tID=LOC_Os01g01010;Name=LOC_Os01g01010;Note=TBC%20domain%20containing%20protein%2C%20expressed\n", "Chr1\tMSU_osa1r7\tgene\t11218\t12435\t.\t+\t.\tID=LOC_Os01g01019;Name=LOC_Os01g01019;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t12648\t15915\t.\t+\t.\tID=LOC_Os01g01030;Name=LOC_Os01g01030;Note=monocopper%20oxidase%2C%20putative%2C%20expressed\n", "Chr1\tMSU_osa1r7\tgene\t16292\t20323\t.\t+\t.\tID=LOC_Os01g01040;Name=LOC_Os01g01040;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t22841\t26971\t.\t+\t.\tID=LOC_Os01g01050;Name=LOC_Os01g01050;Note=R3H%20domain%20containing%20protein%2C%20expressed\n", "Chr1\tMSU_osa1r7\tgene\t27136\t28651\t.\t+\t.\tID=LOC_Os01g01060;Name=LOC_Os01g01060;Note=40S%20ribosomal%20protein%20S5%2C%20putative%2C%20expressed\n", "Chr1\tMSU_osa1r7\tgene\t29818\t34493\t.\t+\t.\tID=LOC_Os01g01070;Name=LOC_Os01g01070;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t35581\t41180\t.\t+\t.\tID=LOC_Os01g01080;Name=LOC_Os01g01080;Note=decarboxylase%2C%20putative%2C%20expressed\n", "Chr1\tMSU_osa1r7\tgene\t44595\t47526\t.\t+\t.\tID=LOC_Os01g01090;Name=LOC_Os01g01090;Note=retrotransposon%20protein%2C%20putative%2C%20unclassified%2C%20expressed\n", "Chr1\tMSU_osa1r7\tgene\t47856\t53412\t.\t-\t.\tID=LOC_Os01g01100;Name=LOC_Os01g01100;Note=retrotransposon%20protein%2C%20putative%2C%20Ty3-gypsy%20subclass%2C%20expressed\n", "grep: write error: Broken pipe\n" ] } ], "source": [ "#1\n", "grep -e \"\\sgene\\s\" all.gff3 | head" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "55986\n" ] } ], "source": [ "#1\n", "grep -e \"\\sgene\\s\" all.gff3 -c\n", "grep -e \"\\sgene\\s\" all.gff3 > all_genes.gff3" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "55986 all_genes.gff3\n" ] } ], "source": [ "wc -l all_genes.gff3" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "wget https://itrop.ird.fr/trainingts/qtl.bed" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "cat qtl.bed" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### Recover sequences of interest from a gff (capture seq)\n", "\n", "We want to extract sequences for each genomic region using `bedtools getfasta`or `fastaFromBed`\n", "\n", "`fastFromBed -fi sequences.fasta -bed bed file`\n", "\n", "Link : https://bioweb.pasteur.fr/docs/modules/bedtools/2.17.0/content/fastafromBed.html" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Tool: bedtools getfasta (aka fastaFromBed)\n", "Version: v2.30.0\n", "Summary: Extract DNA sequences from a fasta file based on feature coordinates.\n", "\n", "Usage: bedtools getfasta [OPTIONS] -fi -bed \n", "\n", "Options: \n", "\t-fi\t\tInput FASTA file\n", "\t-fo\t\tOutput file (opt., default is STDOUT\n", "\t-bed\t\tBED/GFF/VCF file of ranges to extract from -fi\n", "\t-name\t\tUse the name field and coordinates for the FASTA header\n", "\t-name+\t\t(deprecated) Use the name field and coordinates for the FASTA header\n", "\t-nameOnly\tUse the name field for the FASTA header\n", "\t-split\t\tGiven BED12 fmt., extract and concatenate the sequences\n", "\t\t\tfrom the BED \"blocks\" (e.g., exons)\n", "\t-tab\t\tWrite output in TAB delimited format.\n", "\t-bedOut\t\tReport extract sequences in a tab-delimited BED format instead of in FASTA format.\n", "\t\t\t- Default is FASTA format.\n", "\t-s\t\tForce strandedness. If the feature occupies the antisense,\n", "\t\t\tstrand, the sequence will be reverse complemented.\n", "\t\t\t- By default, strand information is ignored.\n", "\t-fullHeader\tUse full fasta header.\n", "\t\t\t- By default, only the word before the first space or tab \n", "\t\t\tis used.\n", "\t-rna\tThe FASTA is RNA not DNA. Reverse complementation handled accordingly.\n", "\n" ] }, { "ename": "", "evalue": "1", "output_type": "error", "traceback": [] } ], "source": [ "fastaFromBed " ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "cd /home/jovyan/SV_DATA/REF" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "#index ref required by fastaFromBed\n", "samtools faidx all.chrs.con" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "fastaFromBed -fi all.chrs.con -bed qtl.bed -fo qtl.fasta" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "6 qtl.fasta\n" ] } ], "source": [ "wc -l qtl.fasta" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "# Convert fasta generated by fastafrom bed (with one line that contained all the sequence extracted for each locus)\n", "# into a fasta with 60 pb per line\n", "seqtk seq -l 60 qtl.fasta > qtl.formatseqtk.fasta" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "9919 qtl.formatseqtk.fasta\n" ] } ], "source": [ "wc -l qtl.formatseqtk.fasta" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr1:2615965-2700945\n", "CAATTAAAATTCAGGTTTAATAGGAGCAAGAAGGCAGGATGAACGAGGAGTGCCCTAGCG\n", "AATACACCCCTTACCACTCCCCCTTCTCCTTTTCTAGCCTCCCACCCTCATCAATGTTGG\n", "\u001b[36m\u001b[K--\u001b[m\u001b[K\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr1:9201049-9467749\n", "TCACCCTAGGGCATAGCCCTGCGCCCCTATCCAAGCCGTTATCTCTATCCGTCGTCGTCG\n", "CTCTTGTCGCCGTCGCGTCGCCATCTTGTCGCCCCCGGTTGAGACCGCTGGGTAATGCGG\n", "\u001b[36m\u001b[K--\u001b[m\u001b[K\n", "\u001b[01;31m\u001b[K>\u001b[m\u001b[KChr4:14902955-15146157\n", "AGTACACCTTTCCCAACTTTCAGAGGATAAAGGGATGCATTCTCAAGACCTCCCAGTCCA\n", "AGGCTAGATAGGGCATTTCTAGATAGGGCATTTCTAATTGTTTAGTTCACAGTATAAAAC\n" ] } ], "source": [ "# print the line with > and the two following lines\n", "grep \">\" qtl.formatseqtk.fasta -A 2" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### Compare two files \n", "\n", "We want to extract annotation information (all_gene.gff3) for each genomic region (qtl.bed) using `bedtools intersect`\n", "\n", "\n", "Link : https://bedtools.readthedocs.io/en/latest/content/tools/intersect.html\n" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Chr1\tMSU_osa1r7\tgene\t2613656\t2623196\t.\t-\t.\tID=LOC_Os01g05510;Name=LOC_Os01g05510;Note=expressed%20protein\tChr1\t2615965\t2700945\t7231\n", "Chr1\tMSU_osa1r7\tgene\t2629374\t2630050\t.\t+\t.\tID=LOC_Os01g05520;Name=LOC_Os01g05520;Note=expressed%20protein\tChr1\t2615965\t2700945\t677\n", "Chr1\tMSU_osa1r7\tgene\t2632554\t2638081\t.\t+\t.\tID=LOC_Os01g05530;Name=LOC_Os01g05530;Note=expressed%20protein\tChr1\t2615965\t2700945\t5528\n", "Chr1\tMSU_osa1r7\tgene\t2640937\t2644665\t.\t-\t.\tID=LOC_Os01g05540;Name=LOC_Os01g05540;Note=expressed%20protein\tChr1\t2615965\t2700945\t3729\n", "Chr1\tMSU_osa1r7\tgene\t2651134\t2651589\t.\t-\t.\tID=LOC_Os01g05550;Name=LOC_Os01g05550;Note=expressed%20protein\tChr1\t2615965\t2700945\t456\n", "Chr1\tMSU_osa1r7\tgene\t2652648\t2655639\t.\t+\t.\tID=LOC_Os01g05560;Name=LOC_Os01g05560;Note=expressed%20protein\tChr1\t2615965\t2700945\t2992\n", "Chr1\tMSU_osa1r7\tgene\t2660822\t2661301\t.\t+\t.\tID=LOC_Os01g05570;Name=LOC_Os01g05570;Note=expressed%20protein\tChr1\t2615965\t2700945\t480\n", "Chr1\tMSU_osa1r7\tgene\t2663539\t2664072\t.\t-\t.\tID=LOC_Os01g05580;Name=LOC_Os01g05580;Note=Exo70%20exocyst%20complex%20subunit%20family%2C%20putative%2C%20expressed\tChr1\t2615965\t2700945\t534\n", "Chr1\tMSU_osa1r7\tgene\t2665412\t2666458\t.\t-\t.\tID=LOC_Os01g05585;Name=LOC_Os01g05585;Note=metallothionein%2C%20putative%2C%20expressed\tChr1\t2615965\t2700945\t1047\n", "Chr1\tMSU_osa1r7\tgene\t2666504\t2667785\t.\t+\t.\tID=LOC_Os01g05590;Name=LOC_Os01g05590;Note=retrotransposon%20protein%2C%20putative%2C%20unclassified%2C%20expressed\tChr1\t2615965\t2700945\t1282\n" ] } ], "source": [ "# -wo\tWrite the original A and B entries plus the number of base pairs of overlap between the two features. \n", "# Only A features with overlap are reported\n", "bedtools intersect -a all_genes.gff3 -b qtl.bed -wo | head " ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Chr1\tMSU_osa1r7\tgene\t2613656\t2623196\tID=LOC_Os01g05510;Name=LOC_Os01g05510;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t2629374\t2630050\tID=LOC_Os01g05520;Name=LOC_Os01g05520;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t2632554\t2638081\tID=LOC_Os01g05530;Name=LOC_Os01g05530;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t2640937\t2644665\tID=LOC_Os01g05540;Name=LOC_Os01g05540;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t2651134\t2651589\tID=LOC_Os01g05550;Name=LOC_Os01g05550;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t2652648\t2655639\tID=LOC_Os01g05560;Name=LOC_Os01g05560;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t2660822\t2661301\tID=LOC_Os01g05570;Name=LOC_Os01g05570;Note=expressed%20protein\n", "Chr1\tMSU_osa1r7\tgene\t2663539\t2664072\tID=LOC_Os01g05580;Name=LOC_Os01g05580;Note=Exo70%20exocyst%20complex%20subunit%20family%2C%20putative%2C%20expressed\n", "Chr1\tMSU_osa1r7\tgene\t2665412\t2666458\tID=LOC_Os01g05585;Name=LOC_Os01g05585;Note=metallothionein%2C%20putative%2C%20expressed\n", "Chr1\tMSU_osa1r7\tgene\t2666504\t2667785\tID=LOC_Os01g05590;Name=LOC_Os01g05590;Note=retrotransposon%20protein%2C%20putative%2C%20unclassified%2C%20expressed\n", "cut: write error: Broken pipe\n" ] } ], "source": [ "# select in the result only some column : from 1st column to 5th plus 9th column\n", "bedtools intersect -a all_genes.gff3 -b qtl.bed -wo | cut -f1-5,9 | head" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Chr1\tMSU_osa1r7\tgene\t2613656\t2623196\tID=LOC_Os01g05510;Name=LOC_Os01g05510;Note=expressed%20protein\t2615965\t2700945\n", "Chr1\tMSU_osa1r7\tgene\t2629374\t2630050\tID=LOC_Os01g05520;Name=LOC_Os01g05520;Note=expressed%20protein\t2615965\t2700945\n", "Chr1\tMSU_osa1r7\tgene\t2632554\t2638081\tID=LOC_Os01g05530;Name=LOC_Os01g05530;Note=expressed%20protein\t2615965\t2700945\n", "Chr1\tMSU_osa1r7\tgene\t2640937\t2644665\tID=LOC_Os01g05540;Name=LOC_Os01g05540;Note=expressed%20protein\t2615965\t2700945\n", "Chr1\tMSU_osa1r7\tgene\t2651134\t2651589\tID=LOC_Os01g05550;Name=LOC_Os01g05550;Note=expressed%20protein\t2615965\t2700945\n", "Chr1\tMSU_osa1r7\tgene\t2652648\t2655639\tID=LOC_Os01g05560;Name=LOC_Os01g05560;Note=expressed%20protein\t2615965\t2700945\n", "Chr1\tMSU_osa1r7\tgene\t2660822\t2661301\tID=LOC_Os01g05570;Name=LOC_Os01g05570;Note=expressed%20protein\t2615965\t2700945\n", "Chr1\tMSU_osa1r7\tgene\t2663539\t2664072\tID=LOC_Os01g05580;Name=LOC_Os01g05580;Note=Exo70%20exocyst%20complex%20subunit%20family%2C%20putative%2C%20expressed\t2615965\t2700945\n", "Chr1\tMSU_osa1r7\tgene\t2665412\t2666458\tID=LOC_Os01g05585;Name=LOC_Os01g05585;Note=metallothionein%2C%20putative%2C%20expressed\t2615965\t2700945\n", "Chr1\tMSU_osa1r7\tgene\t2666504\t2667785\tID=LOC_Os01g05590;Name=LOC_Os01g05590;Note=retrotransposon%20protein%2C%20putative%2C%20unclassified%2C%20expressed\t2615965\t2700945\n", "cut: write error: Broken pipe\n" ] } ], "source": [ "bedtools intersect -a all_genes.gff3 -b qtl.bed -wo | cut -f1-5,9,11,12 | head" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [], "source": [ "bedtools intersect -a all_genes.gff3 -b qtl.bed -wo | cut -f1-5,9,11,12 > qtl.intesect.genes.bed" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "94 qtl.intesect.genes.bed\n" ] } ], "source": [ "# count the number of genes in the three QTL\n", "wc -l qtl.intesect.genes.bed" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2023-10-03 13:33:53-- https://itrop.ird.fr/trainingts/notebook/DAY3-4/Day3_Mapping_Teaching_CORR.ipynb\n", "Resolving itrop.ird.fr (itrop.ird.fr)... 91.203.35.184\n", "Connecting to itrop.ird.fr (itrop.ird.fr)|91.203.35.184|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 85197 (83K)\n", "Saving to: ‘Day3_Mapping_Teaching_CORR.ipynb’\n", "\n", "Day3_Mapping_Teachi 100%[===================>] 83.20K --.-KB/s in 0.03s \n", "\n", "2023-10-03 13:33:53 (3.05 MB/s) - ‘Day3_Mapping_Teaching_CORR.ipynb’ saved [85197/85197]\n", "\n" ] } ], "source": [ "# Go into the correct directory to download notebook - cd command\n", "mkdir -p /home/jovyan/NOTEBOOK/DAY3-4\n", "cd /home/jovyan/NOTEBOOK/DAY3-4\n", "wget https://itrop.ird.fr/trainingts/notebook/DAY3-4/Day3_Mapping_Teaching_CORR.ipynb --no-check-certificate" ] } ], "metadata": { "celltoolbar": "Format de la Cellule Texte Brut", "kernelspec": { "display_name": "Bash", "language": "bash", "name": "bash" }, "language_info": { "codemirror_mode": "shell", "file_extension": ".sh", "mimetype": "text/x-sh", "name": "bash" }, "rise": { "enable_chalkboard": true } }, "nbformat": 4, "nbformat_minor": 4 }