clean csv columns

 pip install csvkit

csvcut -c <column1>,<column2> file.csv

Getting external facing IP

 * curl ifconfig.me

Generating your own certificate

* openssl req -newkey rsa:2048 -nodes -keyout myCompany.key -out myCompany.csr
* openssl x509 -signkey myCompany.key -in myCompany.csr -req -days 365 -out myCompany.crt

Get versions of python modules installed

 pip freeze | egrep 'pandas|pandas_market_calendars'

if you want to install a specific version:

pip install pandas_market_calendars==4.5.1

ssh banners

 This command will output SSH banners from the server as pseudo-terminal is not allocated.

#!/bin/bash

ssh <user>@<server> << EOF

echo 'hi A'

EOF

In order to remove the banner, we can either change the command as follows:

#!/bin/bash

ssh <user>@<server> 'exec bash' << EOF

echo 'hi A'

EOF

or easiest way is 

touch ~/.hushlogin

Zip file exploration

 in data exploration, when you have a lot of zip files, you need a way to look at the cotents with extracting each zip file.

• zcat file.log.zip | less
• unzip -p archive.zip file1.txt | less

OR the best way

• vim archive.zip

VSCode (Not Visual Studio)

 

• Shortcuts:
• Ctrl+P - Open a file
• Ctrl+Alt+F - Search for a file in Explorer

• Extensions:
• Remote Explorer - SSH
• 
  
• Git Lense - Git integration
• Data wrangler - Table view of data frames
• Zip Tools - Exploring zip files in a data set
• Windsurf / Codium - Auto Completion
• R-debugger via command line : install.packages("vscDebugger", repos = "https://manuelhentschel.r-universe.dev") # r-debugger vscode extension

Sending email from Python

 Can setup SMTP inside python itself but for better protection, setup sendmail with SMTP as in other post and then:

import subprocess

import os

import argparse

from email.mime.text import MIMEText

from email.mime.multipart import MIMEMultipart

from email.mime.base import MIMEBase

from email import encoders

def send_file_as_email(file_path, to, subject):

    """Sends an HTML file as the email body and also as an attachment."""

   

    # If subject is None, use the base file name

    if subject is None:

        subject = os.path.basename(file_path)

    # Create the email with mixed content (text + attachments)

    message = MIMEMultipart("mixed")

    message["Subject"] = subject

    message["To"] = to

    # Read HTML content and attach as email body

    with open(file_path, "r") as file:

        html_content = file.read().strip()

    html_body = MIMEText(html_content, "html")

    message.attach(html_body)

    # Attach the file as an attachment

    with open(file_path, "rb") as file:

        attachment = MIMEBase("application", "octet-stream")

        attachment.set_payload(file.read())

    # Encode to base64

    encoders.encode_base64(attachment)

    attachment.add_header("Content-Disposition", f'attachment; filename="{os.path.basename(file_path)}"')

    message.attach(attachment)

    # Send email using sendmail

    process = subprocess.Popen(["/usr/sbin/sendmail", "-t", "-i"], stdin=subprocess.PIPE)

    process.communicate(message.as_bytes())

    print("Email sent successfully!")

def main():

    """Parses command-line arguments and sends an email."""

   

    parser = argparse.ArgumentParser(description="Send an HTML file as an email body & attachment.")

    parser.add_argument("-f", "--file", required=True, help="Path to the HTML file")

    parser.add_argument("-t", "--to", default="harshal.patil@firl.com", help="Recipient email address (default: harshal.patil@firm.com)")

    parser.add_argument("-s", "--subject", default=None, help="Email subject (default: file name)")

    args = parser.parse_args()

    send_file_as_email(args.file, args.to, args.subject)

if __name__ == "__main__":

    main()

Bloomberg BQL

 BQL is not part of daily limit. So we can use those to get the data even when limit is hit:

=@BQL.QUERY("get(" & TEXTJOIN(",", TRUE, B1:E1) & ") for(members('RAY Index'))","cols=7;rows=5919")

where cells B1 to E1 have fields as follows:

market_cap px_volume 30_day_average_volume_at_time factor_expected_daily_volume

Python Selenium Webdriver sample program

 this is a sample program using selenium to click a button:

from selenium import webdriver

from selenium.webdriver.common.by import By

from selenium.webdriver.chrome.service import Service

from webdriver_manager.chrome import ChromeDriverManager

import time

# Set up Chrome options

chrome_options = webdriver.ChromeOptions()

chrome_options.add_argument("--headless")  # Run in headless mode (no UI)

chrome_options.add_argument("--disable-gpu")  # Disable GPU acceleration for headless mode

chrome_options.add_argument("--no-sandbox")  # Disable sandboxing for headless mode

chrome_options.add_argument("--disable-dev-shm-usage")  # Fix shared memory issues

# Set up ChromeDriver using Service (Selenium 4)

service = Service(ChromeDriverManager().install())

# Initialize WebDriver (Chrome) with the path to your ChromeDriver

driver = webdriver.Chrome(service=service, options=chrome_options)

try:

    # Open the website

    driver.get("http://omsprodapp:9900/oms/f#ListedWrapper")

    

    # Wait for the page to load (you might need to adjust this)

    time.sleep(5)

    # Find the "Blotter All" button by its ID and click it

    blotter_button = driver.find_element(By.ID, "blotAllButt")

    blotter_button.click()

    # Wait for the action to complete (adjust the sleep if needed)

    time.sleep(2)

    print("Successfully clicked 'Blotter All'!")

finally:

    # Close the browser after the task is completed

    driver.quit()

logger.sh

 #!/bin/bash

source /home/$LOGNAME/.env

# Ensure at least two arguments are provided

if [ "$#" -lt 2 ]; then

    echo "Usage: $0 <command> <file_path> [additional_args...]"

    exit 1

fi

# Extract filename from the second argument

filename=$(basename "$2")

# Define log directory and log file path

log_dir="/stage/$LOGNAME/logs/$filename"

log_file="$log_dir/$(date +%d).log"

# Create log directory if it doesn't exist

mkdir -p "$log_dir"

# Redirect STDOUT and STDERR to the log file (no output on terminal)

exec >> "$log_file" 2>&1

echo "Executing "$@" at `date`"

# Execute the provided command

eval "$@"

Windows 11 Pro cannot grab window when maximized

 Win + R -> "control access.cpl"

Make the mouse easier to use

UnCheck "Prevent windows from being automatically arranged"

Loading env file in bash

 If you want to load .env file before running a command in bash, we can use following utility. (Can use the just the command as well.)

14:54:00 harshal@HP:~$ set -a; source ~/.env; set +a; CMD

or

14:55:42 harshal@HP:~$ cat ~/load_dot_env.sh

set -a; source ~/.env; set +a

14:55:47 harshal@HP:~$ source ~/load_dot_env.sh 

14:55:49 harshal@HP:~$ 

 

Citrix workspace on Ubuntu 2404

Typically, we start Citrix with this command:

/opt/Citrix/ICAClient/selfservice

But I was getting this error :

harsshal@HP:~$ /opt/Citrix/ICAClient/selfservice 

/opt/Citrix/ICAClient/selfservice: error while loading shared libraries: libwebkit2gtk-4.0.so.37: cannot open shared object file: No such file or directory

But followed these steps and then it worked:

sudo add-apt-repository -y "deb http://gb.archive.ubuntu.com/ubuntu jammy main"

sudo apt update

sudo apt install libwebkit2gtk-4.0-dev

To enable middle click, Change the configuration file  :

sudo vim /opt/Citrix/ICAClient/config/All_Regions.ini:

MouseSendsControlV=False 

AutoHotKey programs

volume.ahk

#Requires AutoHotkey v2.0

Volume_Down::SoundSetVolume "-10"

Volume_Up::SoundSetVolume "+10"

Volume_Mute::SoundSetMute "-1"

mouse.ahk

#Requires AutoHotkey v2.0

waitTime := 5 ; Number of minutes to wait

Loop {

    Sleep waitTime * 60 * 1000

    MouseMove 10, 0, 20, "R"

    MouseMove -10, 0, 20, "R"

}

blotter.ahk

#Requires AutoHotkey v2.0  ; Ensure the script runs with AutoHotkey v2

SetTimer RunCommand, 3600000  ; Set a timer to run every 1 hour (3600000 milliseconds)

RunCommand()

{

    currentTime := FormatTime("HH")  ; Corrected FormatTime call

    if ((currentTime >= 7 and currentTime <= 15) or currentTime = 17)  ; Check if it's between 7 AM - 3 PM or exactly 5 PM

    {

        Run("C:/Users/hpatil/AppData/Local/Microsoft/WindowsApps/python.exe c:/Users/hpatil/webdr.py")

    }

}

or

 #Requires AutoHotkey v2.0  ; Ensure the script runs with AutoHotkey v2

SetTimer RunCommand, 60000  ; Run every 60 seconds (1 minute)

RunCommand()

{

    run_times := ["0715", "0815", "1000", "1340", "1500", "1700"]  ; Define allowed times

    currentTime := FormatTime(A_Now, "HHmm")  ; Get current time in HHMM format

    if HasVal(run_times, currentTime)  ; Check if currentTime is in the array

    {

        Run("C:/Users/hpatil/AppData/Local/Microsoft/WindowsApps/python.exe c:/Users/hpatil/webdr.py")

    }

}

HasVal(haystack, needle)

{

    if !IsObject(haystack)  ; Ensure it's an array before looping

        throw Error("Bad haystack!")  ; Use `Error` instead of `Exception`

    for value in haystack

        if (value = needle)

            return true  ; Return true if found

    return false  ; Return false if not found

}

Sending email via command line

Requirements:

sudo apt install mailutils

sudo apt-get install ssmtp

cat /etc/ssmtp/ssmtp.conf

mailhub=email.firm.com:25 <smpt server>

Then there are multiple ways to it 

cat temp

To: harshal.patil@adyne.com 

Subject: Some Notifying Email 

MIME-Version: 1.0 

Content-Type: text/plain 

Body of your email goes here. Hello world

cat temp | sendmail -i -t

OR

 echo -e "To: harshal.patil@firm.com\nSubject: Subject\nContent-type: text/html" | cat - /stage/qspc/projects/sas/tools/sas_slippage_outliers.html | sendmail -i -t

OR

52 07 * * 1-5 echo "To: harshal.patil@firm.com, harshal.patil@firm.com \nSubject: Slippage Outliers $(date +\%F) \nContent-type: text/html\n\n" > /tmp/email_body.txt;cat /stage/qspc/projects/sas/tools/sas_slippage_outliers.html >> /tmp/email_body.txt;cat /tmp/email_body.txt | /usr/sbin/sendmail -i -t > /stage/qspc/logs/sendmail/$(date +\%d).log 2>&1

If wanted to attach as a file (when html contains graphs):

 (cat /stage/qspcadmin/logs/sendmail/intraday_liquidation.header ; uuencode /stage/qspcadmin/projects/sas/tools/intraday_liquidation.html intraday_liquidation.html) | /sbin/sendmail -i -t

where 

$ cat /stage/qspcadmin/logs/sendmail/intraday_liquidation.header
#To: harshal.patil@firm.com, a.b@firm.com
To: harshal.patil@firm.com
Subject: Intraday Strategy Liquidation

if you wanted an image embeded in the bodt:

(

echo "From: sender@example.com"

echo "To: recipient@example.com"

echo "Subject: Embedded Image"

echo "MIME-Version: 1.0"

echo 'Content-Type: multipart/related; boundary="boundary123"'

echo ""

echo "--boundary123"

echo "Content-Type: text/html; charset=UTF-8"

echo ""

echo '<html><body>'

echo '<p>Here is an embedded image:</p>'

echo '<img src="cid:myimage" width="600" height="400">'

echo '</body></html>'

echo ""

echo "--boundary123"

echo "Content-Type: image/jpeg"

echo 'Content-Transfer-Encoding: base64'

echo 'Content-Disposition: inline; filename="image.jpg"'

echo 'Content-ID: <myimage>'

base64 image.jpg

echo "--boundary123--"

) | sendmail -t

Python / pandas last_bday

import pandas as pd

from pandas.tseries.offsets import BDay

today = pd.to_datetime('today')

last_bday = today - BDay(1) 

Python groupby

 There are 2 ways to normal results:

• use agg function

grp_df = df.groupby(['BookLevel4','Symbol','Direction','OrderId'], as_index=False).agg({'TxTime':['min','max','count'], 'FillQuantity':['sum']})

grp_df.columns = [''.join(col).strip() for col in grp_df.columns.values]

• use apply function 
• In case you use more than 1 column
• Wont have to normalize multi-level columns

grp_df = df.groupby(['BookLevel4','Symbol','Direction','OrderId'], as_index=False).apply(

        lambda s: pd.Series({

            "TxTimemin": s["TxTime"].min(),

            "TxTimemax": s["TxTime"].max(),

            "TxTimecount": s["TxTime"].count(),

            "FillQuantitysum": s["FillQuantity"].sum(),

            "FillVWAP": (s['FillPrice'] * s['FillQuantity']).sum() / s['FillQuantity'].sum()

        })

    )

my .bcrc

 scale=6

my .screenrc

All possible variables are listed here:

https://www.gnu.org/software/screen/manual/html_node/String-Escapes.html#String-Escapes

 Here are the contents of my screenrc:

# Change default scrollback value for new windows

defscrollback 10000 # default: 100

# no welcome message

startup_message off

# advertise hardstatus support to $TERMCAP

termcapinfo  * '' 'hs:ts=\E_:fs=\E\\:ds=\E_\E\\'

termcapinfo xterm* ti@:te@

vbell "off"

bindkey "^[OR" prev

bindkey "^[OS" next

hardstatus alwayslastline

hardstatus string '%{= kG}[%{G}%H%? %1`%?%{g}][%= %{= kw}%-w%{+b yk} %n*%t%?(%u)%? %{-}%+w %=%{g}][%{B}%m/%d %{W}%C%A%{g}]'

screen -t notebook 

screen -t local

Docker commands

 Was trying to create a docker for android. Here are the commands I used:

* sudo apt-get install qemu-kvm

* curl -fsSL https://get.docker.com -o get-docker.sh

* sh ./get-docker.sh

*docker run -d -p 6080:6080 -e EMULATOR_DEVICE="Samsung Galaxy S10" -e WEB_VNC=true --device /dev/kvm budtmo/docker-android:emulator_13.0

* sudo chmod 666 /var/run/docker.sock 

(if permission denied in previous step)

* docker ps

* docker stop <NAME>

Date conversion in MySQL

With recent release, MySQL throws error when inserting dates. This is because, by default "strict mode" is enabled. To get rid of the errors, we need to do something as follows:

SHOW VARIABLES LIKE 'sql_mode';

set sql_mode='';

Installing IBC (Interactive Broker Controller)

 This is a nice utility to keep ibkr TWS running. These are the commands :

$ wget https://github.com/IbcAlpha/IBC/releases/download/3.20.0/IBCLinux-3.20.0.zip

$ sudo unzip IBCLinux-3.20.0.zip -d /opt/ibc

$ sudo chmod o+x /opt/ibc/*.sh /opt/ibc/*/*.sh

$ cat ~/ibc/config.ini

     IbLoginId=<username>

     IbPassword=<password>

$ rm IBCLinux-3.20.0.zip

New Linux installation checklist

• sudo apt install ~/Downloads/google-chrome-stable_current_amd64.deb
• sudo apt-get install gnome-browser-connector
• sudo apt install libreoffice
• sudo apt install git-all
• ssh-keygen -t rsa
• sudo snap install code
• sudo apt install python3-venv
• python3 -m venv ~/.venv/ibkr_python

• https://extensions.gnome.org/extension/1160/dash-to-panel/
• Dash-to-panel settings (right click) -> ungroup applications
• settings -> system -> date/time -> show weekday

• Install vscode, extensions- python

git commands

Following are basic git commands which should be sufficient for most of the git use :

git fetch

git pull

git commit

git push

Automated script to commit:

#!/bin/bash

ssh qspcadmin@USAZLINUXQSPCQ01 << EOF

cd /stage/qspcadmin/projects/sas

git fetch

git pull

git add .

git commit -m "Automated Commit"

git push

EOF

Automated bashrc function to commit with good message:

function gpt_commit() {
    diff=$(git diff)
    response=$(curl -s -X POST https://api.openai.com/v1/chat/completions \
      -H "Authorization: Bearer $OPENAI_API_KEY" \
      -H "Content-Type: application/json" \
      -d @- <<EOF
{
      "model": "gpt-4o-mini",
      "messages": [
        {
          "role": "system",
          "content": "You are an assistant that writes concise and informative git commit messages based on the provided git diff."
        },
        {
          "role": "user",
          "content": "$diff"
        }
      ],
      "temperature": 0.3
}
EOF
    )
    commit_message=$(echo "$response" | grep '"content":' | cut -d '"' -f 4)
    echo "Generated commit message: $commit_message"
    git commit -m "$commit_message"
}
 

Security cameras from desktop

 Need to install ffmpeg in order to access rstp streams

* sudp apt install ffmpeg

Once it is installed, you can access the cameras via

* ffplay rtsp://hpatil:<pass>@<ip_address>:554/stream1

More than one ssh key for the github

 * Generate new key:

[18:05:20 harsshal@dell:~/projects]$ ssh-keygen -t ed25519 -C "harsshal@gmail.com"

* Add new key in the ssh config

[18:07:35 harsshal@dell:~/projects]$ cat ~/.ssh/config 

TCPKeepAlive yes

Host 10.234.1.112

  HostName 10.234.1.112

  User harshal

# Default GitHub

Host github.com

  HostName github.com

  User git

  IdentityFile ~/.ssh/id_rsa

Host github-public

  HostName github.com

  User git

  IdentityFile ~/.ssh/id_ed25519.pub

Host github-personal

  HostName github.com

  User git

  IdentityFile ~/.ssh/id_ed25519

* Clone the repo

[18:09:02 harsshal@dell:~/projects]$ git clone git@github-personal:harsshal/icarus_gl.git

Jupyter notebook setup

jupyter notebook password

cat ~/.ipython/profile_default/startup/start.ipy

import os

import dotenv

dotenv.load_dotenv('/home/'+os.environ['LOGNAME']+'/.env',override=True)

import pandas as pd

# Pandas options

pd.options.display.max_columns = None

pd.options.display.max_rows = None

pd.options.display.max_colwidth = None

          pd.options.display.float_format = '{:,.2f}'.format

print("Everything loaded")

jupyter notebook --notebook-dir [LOCATION] --no-browser --ip=192.168.255.X

jupyter nbconvert --execute --to html notebook.ipynb

Installing a module directly from repo

pip install git+https://github.com/blampe/IbPy.git

pip install git+https://github.com/atreyuxtrading/atreyu-backtrader-api.git

command line / bash variable in awk command

 This is how we use a command line / bash variable (named num) in a awk command. We define an awk  variable 'var' with -v option. This also shows use of BEGIN block.

awk -F, -v var=$num 'BEGIN{srand(var)} {print $1","$2","rand()}' file_name 

Change nameserver entries in resolve.conf

 We can directly change the contents of the file but they dont seem to be in effect. So I added Google's nameserver in the file as follows : 

$ sudo cat /etc/resolvconf/resolv.conf.d/base 

nameserver 8.8.8.8

nameserver 8.8.4.4

And then updated the file as follows : 

$ sudo resolvconf -u 

we can also just update /etc/resolve.conf and it will be in effect immediately.

$ sudo sed -i '1 i\nameserver 8.8.8.8\nnameserver 8.8.4.4' /etc/resolv.conf 

To remove the changes :

$ sudo sed -i '1,2d' /etc/resolv.conf 

renaming all the files in a folder

 IF we want to rename all the files with a part of the file we can do something following:

for file in $(ls); do nn=$(echo $file | awk -F'_' '{print($2)}'); mv $file $nn; done

shell loop through dates

 Easy command to loop through the dates : 

for i in 2022-01-{01..31} 2022-02-{01..15}; do echo $i; done

More comprehensive command :

d=2015-01-01
while [ "$d" != 2015-02-20 ]; do 
  echo $d
  d=$(date -I -d "$d + 1 day")
done

tp-link archer t3u on ubuntu

Recently got tp-link T3u mini wifi adapter  but as usual ubuntu did not have its drivers. So executed following commands and it worked perfectly. 

cd /opt 

git clone https://github.com/cilynx/rtl88x2bu.git 

cd rtl88x2bu

VER=$(sed -n 's/\PACKAGE_VERSION="\(.*\)"/\1/p' dkms.conf)

sudo rsync -rvhP ./ /usr/src/rtl88x2bu-${VER}

sudo dkms add -m rtl88x2bu -v ${VER}

sudo dkms build -m rtl88x2bu -v ${VER}

sudo dkms install -m rtl88x2bu -v ${VER}

sudo modprobe 88x2bu

sudo access to ldap user

 Typically, if you dont have sudo and "gain" sudo access, you cant just add yourself to sudo group. In that scenario, we need to reboot in "advanced mode" and gain access to root shell. Once in, you can have following : 

$ sudo cat /etc/sudoers.d/hpatil 

hpatil ALL=(ALL) ALL

and then we will have sudo access. 

Bootable USB

In case your laptop has good old booting system and not advanced system like UEFI, you can use following command to create bootable USB. In case your laptop is fancy, you will have to use software like Rufus which works only on "amazing" windows.

sudo dd bs=4M if=~/Downloads/ of=/dev/sdc1 status=progress oflag=sync

you can find out /dev/sdc1 via df command.

Highlight mouse with win + F5

When you have multiple monitors, and we need to search for a mouse, it becomes difficult to see where its hiding. Windows had a good feature to highlight mouse when you press Ctrl, but linux typically dont have that. So I used following workaround to achieve same effect in i3.

sudo apt install libx11-dev libxcomposite-dev libxdamage-dev libxrender-dev
mkdir ~/git
cd git
git clone https://github.com/Carpetsmoker/find-cursor.git
cd find-cursor
make
echo "bindsym $mod+F5 exec /home/maahi/git/find-cursor/find-cursor -of" >> ~/.config/i3/config

docker commands

A while ago, I was playing around with docker. Here is the collection of all the commands I used. I used following to install dockers:

sudo echo 'deb https://apt.dockerproject.org/repo debian-stretch main' > /etc/apt/sources.list.d/dockertemp.list
sudo apt-get update
sudo apt-get install docker-engine
sudo rm /etc/apt/sources.list.d/dockertemp.list sudo service docker start

 Once installation was done, these were basic commands I used :

https://sakuli.readthedocs.io/en/v1.0.0/docker-containers/
sudo docker pull harsshal/tws:first
sudo docker run -it -p 5901:5901 -p 8000:6901 harsshal/tws:first
sudo docker run -it -p 5901:5901 -p 8000:6901 -exec -u 0 harsshal/tws:first /bin/bash
sudo docker run -it -p 5901:5901 -p 8000:6901 -e "VNC_PW=ibtrader" harsshal/tws:first
sudo docker ps -a
sudo docker start
sudo docker commit harsshal/tws:second
sudo docker push harsshal/tws:second
sudo docker images
sudo docker ps -a | awk '{print $1}' | xargs sudo docker rm

Display all dataframe lines in ipython / jupyter notebook

Sometimes when you want to see all lines in ipython, either in IDE or in notebook, it wont show you because of in-build options. Here are those options, in case you want to change that :

import pandas as pd
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 1000)

Vertica creating a temporary list of numbers

I just wanted to quickly check what is the behavior of NULL in analytics functions like median.
So rather than creating a table, I  wanted to pass a list of numbers and see what happens. This is how I achieved it :

SELECT median(val) over() FROM (
SELECT * from (SELECT 1 as val, 1 as row
UNION SELECT 2, 2
UNION SELECT NULL,3
UNION SELECT NULL,4
UNION SELECT 4,5)temp
)temp2;

i3 layout reload configuration

Recently started playing around with i3. I liked it so far. It can open all the applications where you need after a restart. After learning how to split the screens etc, get to your desired configuration. Once fixed, execute following : 

i3-save-tree > i3.json

We need to edit json file so that "class" entries in "swallows" sections are un-commented.

Enter following in config "~/.config/i3/config" :

exec load_i3_layout.sh

Where as contents of the shell scripts are :

i3-msg 'workspace 1; append_layout i3.json'
i3-msg 'exec /usr/bin/chromium-browser'
i3-msg 'exec /usr/bin/dbeaver'
i3-msg 'exec /snap/bin/pycharm-community'
i3-msg 'exec /usr/bin/nautilus'
i3-msg 'exec /usr/bin/gnome-terminal'
i3-msg 'exec /usr/bin/chromium-browser'

This will open up all the windows at your needed locations.

awk delimiters

I have been using awk for a while and I had a use case where I wanted to use multiple delimiter, ' ' being one of them.
If its just space, we can use :

awk -F' ' '{print $2}'

If its bunch of characters, we can use :

awk -F'[.:]' '{print $2}' 

But with space, we need to use as follows :

awk -F'[[:space:].]+' '{print $2}'

I have not been able to figure out to associate '+' only to space. But for now this will server the purpose as long as I can combine multiple '.'

Some other classes are:

[[:blank:][:cntrl:]] 

[ \t,:;]

Python notebook shortcuts

Recently started using a lot of notebook and keep struggling to remember shortcuts. So posting here so that I'll remember few important ones.


Ctrl + Enter : Execute current cell
Ctrl + / : multi-line comment
Ctrl + Shift + - : Split cell
Shift + m : Merge cells
a : Above current cell, create a cell
b : Below current cell, create a cell

Linux mint suspend after more than 1 hour

Trying to determine ideal time for suspending the machine but found that maximum time Linux mint allows before it can suspend the machine is 1 hour. After researching a bit, found that we can extend this time. These are the commends :

1. sudo apt-get install dconf-editor
2. Open Menu->dcong-editor
3. In dconf-editor, org > cinnamon > settings-daemon > plugins > power and manually set the value for the "sleep-inactive-ac-timeout" (it's in seconds)

Installing new packages behind proxy

There are 2 main ways to do this :

pip install pandas
python -m pip install pandas --proxy proxy..com:3128

Second option has many other parameters which can be passed.

For installing via notebook, we can do as follows :

import sys
!{sys.executable} -m pip install numpy

!{sys.executable} -m pip list --user 

Converting screen into tmux ( terminal multiplexer )

We have beutiful program called
tmux
which allows us to see multiple terminal windows at the same time. We can covert screen in doing something similar.

To split vertically: ctrl + a then |.
To split horizontally: ctrl + a then S (uppercase 's').
To unsplit: ctrl + a then Q (uppercase 'q').
To switch from one to the other: ctrl + a then tab

We can save layouts in screenrc so that we wont have to keep doing this as well.

Shell keyboard shortcuts

On Linux shell, imagine you are executing a long command and then you remember that you need to execute other command first.

Just found out an awesome feature where you can put a command in buffer (Ctrl+U), execute another command and then retrieve the buffer(Ctrl+Y).

Awesome!

Of course, Ctrl+A (Ctrl+A+A when you are in screen) to take you to the start of the line and Crtl+E to take you to end of the line.

Linux mint buzzing sound when using headphones

I had annoying buzzing sound coming in from speakers whenever I used to insert headphones. After few trial and errors I found following solution :

• Open terminal and type

Alsamixer 

• After that select the sound card (usually PCH)
• Go to far right until optiona

Auto-mute mode 

• press up arrow to make it disabled.  

sqllite commandline history via up/down keys

I found very annoying problem with sqlite CLI. When you press up/down arrow keys, it prints ASCI characters of the keys rather that showing historical commands. When I googled, I realised that we can either link readline library when compiling sqlite or do:

sudo apt-get install rlwraprlwrap sqlite3 database.db

worked like a charm for me.

increasing brightness on LXDE display on lubunut

LXDE seems to have 2 brightness controls. Both of them are found in locations: /sys/class/backlight/

For me it was :

harsshal@dell:~$ ll /sys/class/backlight/ 

lrwxrwxrwx  1 root root 0 Mar 18 12:58 acpi_video0 -> ../../devices/pci0000:00/0000:00:02.0/backlight/acpi_video0/lrwxrwxrwx  1 root root 0 Mar 18 12:58 intel_backlight -> ../../devices/pci0000:00/0000:00:02.0/drm/card0/card0-LVDS-1/intel_backlight/

Both locations have max_brightness file :

harsshal@dell:~$ ll /sys/class/backlight/*/max_brightness -r--r--r-- 1 root root 4096 Mar 18 12:58 /sys/class/backlight/acpi_video0/max_brightness-r--r--r-- 1 root root 4096 Mar 18 12:58 /sys/class/backlight/intel_backlight/max_brightness

harsshal@dell:~$ cat /sys/class/backlight/*/max_brightness 7255000

So typically intel_backlight will be controlled by all your default brightness controller, but acpi_video0 will need manual changes when you need to change it:

harsshal@dell:~$ echo 7 | sudo tee /sys/class/backlight/acpi_video0/brightness 7

Other methods of increasing brightness :

harsshal@dell:~$ xbacklight -set 60

Or

harsshal@dell:~$ xrandr -q | grep -w connectedLVDS-1 connected 1280x800+0+0 (normal left inverted right x axis y axis) 290mm x 180mmharsshal@dell:~$ xrandr --output LVDS-1 --brightness 1

startup scripts in windows to keep screen active

I wrote a script to keep my comupter not get locked. script goes as follows:

set WshShell = WScript.CreateObject("WScript.Shell")

x = 1

do while x = 1 'Loop forever and ever and ever and ever

WshShell.SendKeys "%^" 'SEND CTL + ALT

WScript.Sleep 290000 '<5 MINS

loop

I placed this script at location C:\Users\username\AppData\Roaming\Microsoft\Windows\Start Menu\Programs\Startup
and no more locking up the screen....

Adding right click options

I add my custom scripts to perform certain actions via right-click. To achieve that, we need to do following :

vim ~/.gnome2/nautilus-scripts/terminal

For nemo, we need to have scripts at following locations:

/usr/share/nemo/actions/ for system-wide actions
$HOME/.local/share/nemo/actions/ for user actions

Type the command you want to have after right click. In this example, I do :

gnome-terminal

And make that script executable:

chmod a+x ~/.gnome2/nautilus-scripts/terminal

Citrix receiver on linux version LUbuntu

Started my new year by installing Lubuntu on my old dell laptop. Usually I install via USB, so prepared USB again via

usb-creator-gtk

on my previous installation of linux mint.
Installation always used to hang on certain point on installer and could not figure out why. After googling, came to know that it needs network during installation but does not give any error. So connected LAN cable and got it installed.

First task was to make sure that no issues when connecting to office network. I installed receiver from

https://www.citrix.com/downloads/citrix-receiver.html

After installation, logged in and got an verisign ssl error. So did

sudo cp /usr/share/ca-certificates/mozilla/* /opt/Citrix/ICAClient/keystore/cacerts

One issue is chrome does not open citrix application right away. I need to open it via command line by executable located at

/opt/Citrix/ICAClient/wfica

Creating python executables

I needed to create python executable so that people wont have to rely on my python version.
So these are the steps:
# Install "cx_Freeze"
# Create setup.py

$catsetup.py 

from cx_Freeze import setup, Executable
build_exe_options = {
"includes": ['numpy', 'pandas','matplotlib.backends.backend_qt4agg'],
"packages": [],
"excludes": ['tk','ttk','zmq','boto','tkinter','_gtkagg', '_tkagg', 'bsddb', 'curses', 'pywin.debugger','pywin.debugger.dbgcon', 'pywin.dialogs', 'tcl', 'Tkconstants', 'Tkinter'],
"include_files": []}

setup(
    name = "appName",
    version = "0.1",
    description = "",
    author = "Dengar",
    options = {"build_exe": build_exe_options},
    executables = [Executable("sectorize.py")]
)

#python setup.py build

Alt+Tab not working in remote desktop of citrix receiver on Linux mint 17

I use citrix to connect to office computer. But Alt+Tab wasn't seem to be working when I used remote desktop. I found few solution about HKEY edit solution for windows, but none for linux mint. I did some trial and error and finally found a solution myself.

I found this file in installation directory of ICAClient:

[17:43:43 harsshal@lenovo:~]$ ll /opt/Citrix/ICAClient/config/All_Regions.ini

lrwxrwxrwx 1 root root 37 Nov 8 2013 /opt/Citrix/ICAClient/config/All_Regions.ini -> /etc/icaclient/config/All_Regions.ini

Went to link and started edited this file

[17:44:08 harsshal@lenovo:~]$ sudo vim /etc/icaclient/config/All_Regions.ini

Search for "TransparentKeyPassthrough" in "Virtual Channels" section and add "Remote" in front of it.

[Virtual Channels\Keyboard]
TransparentKeyPassthrough=Remote

Save and restart Remote Desktop Connection

Google CodeJam in python templates

I recently started learning python and designed these programs.

Following program is for http://code.google.com/codejam/contest/351101/dashboard#s=p2


 # -*- coding: utf-8 -*-
"""
Created on Tue Apr  7 15:40:09 2015

@author: hpatil
"""

import sys

def read_file( input_file, output_file):
    sys.stdin = open(input_file)
    if(output_file != '' ):
        sys.stdout = open(output_file,"w+")
    read_input()

def solver(line):
    last_digit=10
    for i in line:
        #print(i)
        #print(len(line))
        digit, mystring = digit2string(i)
        if(last_digit==digit):
            print(' ',end='')
        print(mystring,end='')
        last_digit=digit
    print()
    return

def digit2string(char):
    dial = [
        [' '],
        [],
        ['a','b','c'],
        ['d','e','f'],
        ['g','h','i'],
        ['j','k','l'],
        ['m','n','o'],
        ['p','q','r','s'],
        ['t','u','v'],
        ['w','x','y','z'],
    ]
    mystring = ""  
    for digit in range(len(dial)):
        for repeat in range(len(dial[digit])):
            if(char==dial[digit][repeat]):
                for times in range(repeat+1):
                    mystring+=str(digit)
                return digit,mystring
  
  
def read_input():  
    with sys.stdin as file:
        testCases = int(file.readline())
        for test in range(testCases):
            print("Case #{}: ".format(test+1), end="")
            line = file.readline().strip('\n')
            solver(line)

#read_file('input','')
read_input()


This one is for with integer reading for http://code.google.com/codejam/contest/351101/dashboard


# -*- coding: utf-8 -*-
__author__ = 'hpatil'

import sys

def show_graph(x,y):
    import matplotlib.pyplot as plt

    plt.plot (x, y)
    plt.grid(True)
    plt.show()

def solver(credit, nItems, items):
    for i in range(nItems):
        for j in range(i+1,nItems):
            sum = items[i] + items[j]
            if( sum == credit):
                print(i+1,j+1)
                return

def set_io( input_file, output_file):
    sys.stdin = open(input_file)
    sys.stdout = open(output_file,"w")
   
def read_input():   
    with sys.stdin as file:
        testCases = int(file.readline())
        for test in range(testCases):
            print("Case #{}: ".format(test+1), end="")
            credit = int(file.readline())
            nItems = int(file.readline())
            items = list(map(int,file.readline().split()))
            show_graph(range(len(items)), items)
            solver(credit, nItems, items)

set_io('input','out')
read_input()

Following program is for http://code.google.com/codejam/contest/3214486/dashboard which took me a loooooong time

# -*- coding: utf-8 -*-
"""
Created on Wed Apr  8 13:08:15 2015

@author: hpatil
Solution to problem : http://code.google.com/codejam/contest/3214486/dashboard

"""

lit = [
        #['abcdefg']
        '1111110',
        '0110000',
        '1101101',
        '1111001',
        '0110011',
        '1011011',
        '1011111',
        '1110000',
        '1111111',
        '1111011'
    ]


def display_calci(string):
    if(string=="ERROR!"):
        print(string)
    else:
        display = list("         ")
        print(''.join(display))
        # line 1
        if(string[5] =='1'):
            display[0]='|'
        if(string[0] =='1'):
            display[1]='^'
        if(string[6] =='1'):
            display[2]='_'
        if(string[1] =='1'):
            display[3]='|'
        display[4]='\n'
        if(string[4] =='1'):
            display[5]='|'
        if(string[3] =='1'):
            display[6]='_'
            display[7]='_'
        if(string[2] =='1'):
            display[8]='|'
        print(''.join(display))

def isDigitFeasible(active, disp_digit, digit):
    for led in range(7):
        if(active[led]=='1'):
            if( disp_digit[led] != lit[digit][led]):
                return 0
    return 1

def get_active_leds(active, disp_digits):
    active= list(active)
    count = int(disp_digits[0])
    for digit in range(1,count+1):
        for led in range(len(disp_digits[digit])):
            if(disp_digits[digit][led]=='1' ):
                active[led]='1'
    return ''.join(active)

def led_unused(active, disp_digits, possible_digit):
    not_used = list('1111111')
    active= list(active)
    count = int(disp_digits[0])
    for digit in range(1,count+1):
        possible_digit= (possible_digit+1)%10
        for led in range(len(lit[possible_digit])):
            if(lit[possible_digit][led]=='1' ):
                not_used[led]='0'
    return ''.join(not_used)

def print_digit(active, disp_digits, digit):
    result_string = ""
    unused = led_unused(active, disp_digits, digit)
    for led in range(len(active)):
        if(active[led]=='1'):
            result_string += lit[digit][led]
        else:
            if(lit[digit][led] == '1'):
                if(unused[led]=='1'):
                    return "ERROR!"
                else:
                    result_string += '0'
            else:
                result_string += '0'

    return result_string

def get_possible_digits(active, disp_digit, possible):
    possible_return = [ ]
    for digit in possible:
        if(isDigitFeasible(active, disp_digit, digit)):
            possible_return.append(digit)
    return possible_return

def solver(disp_digits):
    active= list('0000000')
    active = get_active_leds(active,disp_digits)
    if(debug):
        display_calci(active)
    count = int(disp_digits[0])
    possible = [9,8,7,6,5,4,3,2,1,0]
    for digit in range(1,count+1):
        possible = get_possible_digits(active,disp_digits[digit], possible)
        possible = list(map(lambda x: (x-1) % 10, possible))
    if(debug):
        print(possible)
    possible_soln = list(map(lambda x: print_digit(active,disp_digits,x), possible))
    if(debug):
        print(possible_soln)
    soln_set = set(possible_soln)
    if(len(soln_set) != 1):
        return "ERROR!"
    else:
        return list(soln_set)[0]

def read_input():
    with sys.stdin as file:
        nTests = int(file.readline())
        for tests in range (1, nTests+1):
            print("Case #" + str(tests)+": ", end="")
            disp_digits = file.readline().split('\n')[0].split(' ')
            if(debug):
                print(disp_digits[1:])
                list(map(lambda x: display_calci(x), disp_digits[1:]))
                print("-----------")
            answer_string = solver(disp_digits)
            if(debug):
                display_calci(answer_string)
            print(answer_string)

import sys
debug = int(sys.argv[1])
sys.stdin = open (sys.argv[2])
#sys.stdin = open ('input2')
#sys.stdin = open ('A-small-practice.in')
#sys.stdin = open ('A-large-practice.in')

read_input()