Difference between revisions of "Linux Administration"

From Hawk Wiki
Jump to: navigation, search
(→‎auto generate git commit message with one file name)
(→‎Ubuntu create swap)
 
(103 intermediate revisions by the same user not shown)
Line 5: Line 5:
 
==[[AWS Redhat RHEL Install LAMP]]==
 
==[[AWS Redhat RHEL Install LAMP]]==
 
==[[Install VPN PPTPD on RHEL 7]]==
 
==[[Install VPN PPTPD on RHEL 7]]==
 +
==[[Install OPen VPN client on ubuntu]]==
 +
 +
<pre>
 +
#setup /etc/openvpn/openvpn.conf
 +
#/etc/openvpn/auth.txt
 +
 +
#manually start
 +
sudo openvpn --config /etc/openvpn/openvpn.conf &
 +
 +
</pre>
  
 
==[[AWS Redhat RHEL Install Ruby and Rails]]==
 
==[[AWS Redhat RHEL Install Ruby and Rails]]==
  
 
==[[Ubuntu Install GUI]]==
 
==[[Ubuntu Install GUI]]==
 +
 +
== clean up linux disk space==
 +
<pre>
 +
sudo journalctl --vacuum-time=2d
 +
sudo apt-get autoremove --purge
 +
 +
sudo snap set system refresh.retain=2
 +
 +
</pre>
 +
 
==tar and gzip==
 
==tar and gzip==
 
<pre>Archive a set of files:
 
<pre>Archive a set of files:
Line 45: Line 65:
  
 
cd /etc; tar cf – . | (cd /etc.bak; tar xvpf -)
 
cd /etc; tar cf – . | (cd /etc.bak; tar xvpf -)
 +
 +
#parallel zip
 +
# compress
 +
pigz -2 ./abc.vdi
 +
# extract / decompress
 +
pigz -d ./abc.vdi.gz
 
</pre>
 
</pre>
 +
 +
==zip with split and password==
 +
interactive
 +
<pre>zip --encrypt -s 4g -r file.zip files</pre>
 +
Or
 +
<pre>
 +
// recursive zip all folder
 +
// with password
 +
// split 4gb
 +
zip --password password -s 4g -r output.zip input &
 +
// with store only and 1gb split
 +
zip -Z store -r -s 1g output.zip input/
 +
</pre>
 +
 +
==no hangup even if logged out of ssh==
 +
<pre>
 +
nohup bash script.sh &> script.log &
 +
</pre>
 +
 
==DB backup==
 
==DB backup==
 
# mysqldump -u root -p [root_password] [tabdaase_name] > dumpfilename.sql<br>
 
# mysqldump -u root -p [root_password] [tabdaase_name] > dumpfilename.sql<br>
Line 136: Line 181:
 
==List disk usage by directory==
 
==List disk usage by directory==
 
du -h <dir> | grep '[0-9\.]\+G'
 
du -h <dir> | grep '[0-9\.]\+G'
−
==Ubuntu 16.4 create swap==
+
==Ubuntu create swap==
 
<pre>
 
<pre>
 
sudo fallocate -l 1G /swapfile
 
sudo fallocate -l 1G /swapfile
Line 145: Line 190:
 
sudo swapon /swapfile
 
sudo swapon /swapfile
 
sudo swapon --show
 
sudo swapon --show
 +
</pre>
 +
to auto mount
 +
<pre>
 +
echo '/swapfile none swap sw,nofail 0 0' | sudo tee -a /etc/fstab
 
</pre>
 
</pre>
  
 
https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-16-04
 
https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-16-04
 +
 +
=== use entire disk for swap ===
 +
<pre>
 +
sudo mkswap /dev/nvme4n1
 +
sudo swapon /dev/nvme4n1
 +
</pre>
  
 
==How to auto run scripts at system start in Cent OS 7==
 
==How to auto run scripts at system start in Cent OS 7==
Line 184: Line 239:
 
</pre>
 
</pre>
  
−
==auto generate git commit message with one file name==
+
==Prepare extra ~/bin folder for shortcut commands==
−
<pre class="brush:Bash/shell">
+
edit ~/.bash_profile add line
 +
<pre>
 +
export PATH="~/bin/:$PATH"
 +
</pre>
 +
 
 +
==Auto generate git commit message with one file name==
 +
<pre>
 +
touch ~/bin/autocommit && chmod +x ~/bin/autocommit && vim ~/bin/autocommit
 +
</pre>
 +
Paste
 +
<pre>
 
#!/bin/bash
 
#!/bin/bash
 
BRANCH_NAME=$(git symbolic-ref --short HEAD)
 
BRANCH_NAME=$(git symbolic-ref --short HEAD)
Line 198: Line 263:
 
FIRST_FILE=${COMMIT_MSG%%$'\n'*}
 
FIRST_FILE=${COMMIT_MSG%%$'\n'*}
 
git commit -m "[${BRANCH_NAME}] ${FIRST_FILE}"
 
git commit -m "[${BRANCH_NAME}] ${FIRST_FILE}"
 +
</pre>
 +
usage: just type <b>autocommit</b> and it will generate a commit message containing the current branch name and first file changed
 +
 +
===Auto commit and push===
 +
<pre>
 +
git config --global push.default current
 +
echo "autocommit && git push" > ~/bin/autopush
 +
chmod +x ~/bin/autopush
 +
</pre>
 +
usage: just type <b>autopush</b>
 +
==Mount hard drive==
 +
<pre>
 +
#find uuid
 +
sudo blkid
 +
#mount hard drive
 +
sudo mount /dev/sda1 /mnt/ssd1
 +
 +
 +
# auto mount
 +
sudo vim /etc/fstab
 +
#add
 +
UUID=0c18cc09-0028-4cb1-97ef-6222fe477432  /mnt/ssd  ext3    defaults,nofail    0        2
 +
</pre>
 +
Better method
 +
<pre>
 +
DRIVE=sdb1
 +
FOLDER=ssd1
 +
UUID=$(sudo blkid /dev/${DRIVE} -sUUID -ovalue)
 +
echo $UUID
 +
echo "UUID=${UUID} /mnt/${FOLDER} xfs defaults,nofail 0 2" | sudo tee -a /etc/fstab
 +
sudo mount --uuid $UUID /mnt/${FOLDER}
 +
</pre>
 +
 +
==fix corrupt hard drive file system==
 +
<pre>
 +
sudo gdisk /dev/sdb
 +
 +
sudo e2fsck /dev/sdb1
 +
</pre>
 +
 +
==using gparted for over 2tb==
 +
https://www.cyberciti.biz/tips/fdisk-unable-to-create-partition-greater-2tb.html
 +
<pre>
 +
 +
sudo apt-get install -y gparted xfsprogs
 +
 +
DRIVE=sdb
 +
FOLDER=ssd1
 +
 +
sudo parted /dev/${DRIVE}
 +
 +
mklabel gpt
 +
mkpart primary 0G 100%
 +
print
 +
quit
 +
 +
sudo mkfs.xfs /dev/${DRIVE}1
 +
sudo mkdir -p /mnt/${FOLDER}
 +
# mount hard drive
 +
UUID=$(sudo blkid /dev/${DRIVE}1 -sUUID -ovalue)
 +
echo $UUID
 +
echo "UUID=${UUID} /mnt/${FOLDER} xfs defaults,nofail 0 2" | sudo tee -a /etc/fstab
 +
sudo mount --uuid $UUID /mnt/${FOLDER}
 +
 +
</pre>
 +
 +
==Test disk write speed==
 +
<pre>
 +
sudo chown -R $USER:$USER ./
 +
dd if=/dev/zero of=./test1.img bs=1G count=1 oflag=dsync && rm ./test1.img
 +
 +
</pre>
 +
 +
using fio
 +
 +
<pre>
 +
sudo apt-get install fio
 +
 +
 +
fio --filename=test --sync=1 --rw=randread --bs=4k --numjobs=1 \
 +
  --iodepth=4 --group_reporting --name=test --filesize=2G --runtime=300 && rm test
 +
 +
 +
fio --filename=test --sync=1 --rw=randwrite --bs=4k --numjobs=1 \
 +
  --iodepth=4 --group_reporting --name=test --filesize=2G --runtime=300 && rm test
 +
 
 +
 +
</pre>
 +
 +
==Splunk installation==
 +
download splunk
 +
then follow
 +
http://docs.splunk.com/Documentation/Splunk/7.0.3/Installation/InstallonLinux
 +
 +
Change port
 +
<pre>sudo vim $SPLUNK_HOME/etc/system/local/web.conf</pre>
 +
add
 +
<pre>
 +
[settings]
 +
httpport = 8866
 +
</pre>
 +
 +
Configure to start on boot time
 +
http://docs.splunk.com/Documentation/Splunk/latest/Admin/ConfigureSplunktostartatboottime
 +
<pre>
 +
sudo $SPLUNK_HOME/bin/splunk enable boot-start
 +
sudo service splunk start
 +
</pre>
 +
 +
==List file sizes by directory (depth = 1)==
 +
<pre>
 +
du -ah --max-depth=1 | sort -n
 +
</pre>
 +
 +
== Copy files argument list too long ==
 +
<pre>
 +
echo ./* | xargs cp -r --  /target-dir
 +
 +
echo ./* | xargs rsync -avh -- /target-dir
 +
 +
echo ./* | xargs  mv -t /target-dir --
 +
 +
</pre>
 +
 +
== clean up journal ==
 +
<pre>
 +
sudo journalctl --vacuum-time=2days
 +
sudo journalctl --vacuum-size=100M
 +
 +
sudo vim /etc/systemd/journald.conf
 +
# change
 +
# SystemMaxUse=200M
 +
 +
sudo systemctl daemon-reload
 +
 +
</pre>
 +
 +
https://ubuntuhandbook.org/index.php/2020/12/clear-systemd-journal-logs-ubuntu/
 +
 +
==Allow pm2 run via ssh and cron==
 +
<pre>
 +
n=$(which node); \
 +
n=${n%/bin/node}; \
 +
chmod -R 755 $n/bin/*; \
 +
sudo cp -r $n/{bin,lib,share} /usr/local
 +
</pre>
 +
method 2:
 +
<pre>
 +
sudo rm -f /usr/local/bin/pm2 && sudo ln -s $(which pm2) /usr/local/bin/pm2
 +
</pre>
 +
 +
==control cpu fan speed==
 +
see https://askubuntu.com/questions/22108/how-to-control-fan-speed
 +
<pre>
 +
sudo apt-get install fancontrol lm-sensors
 +
sudo sensors-detect
 +
# answer yes to all questions
 +
</pre>
 +
 +
check temp
 +
<pre>sensors</pre>
 +
 +
edit /etc/default/grub
 +
to add acpi_enforce_resources=lax to the
 +
GRUB_CMDLINE_LINUX_DEFAULT
 +
 +
then run <pre>sudo update-grub</pre> reboot
 +
<pre>
 +
sudo service kmod start
 +
sudo pwmconfig
 +
</pre>
 +
view current config
 +
<pre>
 +
/etc/fancontrol
 +
</pre>
 +
finally start fan control
 +
<pre>sudo service fancontrol start</pre>
 +
auto start
 +
<pre>sudo update-rc.d fancontrol enable</pre>
 +
config explaination
 +
 +
MINTEMP: The temperature (°C) at which to SHUT OFF the CPU fan. Efficient CPUs often will not need a fan while idling. Be sure to set this to a temperature that you know is safe. Setting this to 0 is not recommended and may ruin your hardware!
 +
 +
MAXTEMP: The temperature (°C) at which to spin the fan at its MAXIMUM speed. This should be probably be set to perhaps 10 or 20 degrees (°C) below your CPU's critical/shutdown temperature. Setting it closer to MINTEMP will result in higher fan speeds overall.
 +
 +
MINSTOP: The PWM value at which your fan stops spinning. Each fan is a little different. Power tweakers can echo different values (between 0 and 255) to /sys/class/hwmon/hwmon0/device/pwm1 and then watch the CPU fan. When the CPU fan stops, use this value.
 +
 +
MINSTART: The PWM value at which your fan starts to spin again. This is often a higher value than MINSTOP as more voltage is required to overcome inertia.
 +
 +
==raw disk backup==
 +
if: input file
 +
 +
of: output file (image)
 +
<pre>dd if=/dev/mmcblk0 of=/media/usb500/pi_system_torrent.image</pre>
 +
 +
==search for file name or extension==
 +
search for file name or extension
 +
find . -iname "*.jpg"
 +
 +
==delete files by matches==
 +
find . -iname "*.jpg" -delete
 +
===rsync file from remote to local===
 +
<pre>
 +
nohup rsync -avh -e "ssh -x -T -o Compression=no" root@domain.com:/mnt/ssd1/* ./ >~/rsync.log &
 +
</pre>
 +
 +
The above command turn off compression so it will be faster.
 +
 +
===rsync locally===
 +
<pre>
 +
sudo rsync -aqxP /mnt/ssd1/docker/ /nvmex4/docker
 +
</pre>
 +
 +
==search==
 +
grep --color -n -C 1 "search_text" lib/npm.js
 +
 +
==regex==
 +
grep --color -n -e "search_text[abc|abe]" lib/npm.js
 +
 +
==search all files recursive==
 +
grep -HR --color "loan" ./
 +
===backup whole system===
 +
<pre>
 +
sudo su
 +
dd if=/dev/sdc | bzip2 > /mnt/ssdbackup/ubuntuBackUp201803.bz2
 +
bzcat /mnt/ssdbackup/ubuntuBackUp201803.bz2 | dd of=/dev/sdc
 +
</pre>
 +
 +
==[[Grow linux partitions on live system]]==
 +
 +
==Start socks proxy server==
 +
see https://github.com/adegtyarev/docker-dante
 +
use docker-compose
 +
==Install fusion io drive2 driver in ubuntu==
 +
[[fusionio driver]]
 +
 +
==Port Forwarding Simple TCP Traffic==
 +
use rinetd
 +
https://github.com/boutell/rinetd
 +
==rinetd simple tcp traffic forwarding==
 +
This is useful for forwarding traffic to server behind vpn
 +
 +
==Auto restart process if exit==
 +
<pre>
 +
#/bin/bash
 +
RIGID=$(hostname)
 +
echo $RIGID
 +
STATUS=`ps -A | grep gethbsc`
 +
# change channel to slack webhook
 +
CHANNEL="https://hooks.slack.com/services/xxxxxxxx/xxxxxxx"
 +
if [ ! -z "$STATUS" -a "$STATUS" != " " ] ; then
 +
  echo $STATUS
 +
else
 +
  echo "Restart service"
 +
  cd ~/bsc && bash ~/bsc/start.sh
 +
  curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"server=${RIGID} node down\"}" $CHANNEL
 +
fi
 +
</pre>
 +
 +
==RDMA NDS system via infiniband==
 +
this only works in centOS8
 +
 +
on server
 +
 +
change nfs disk array server ib ip to 10.1.100.1
 +
 +
change client server ib ip to 10.1.100.2
 +
<pre>
 +
vim /etc/exports
 +
# edit and add
 +
/mnt/nfs        10.1.100.0/24(rw,sync,no_subtree_check,insecure,no_root_squash)
 +
/mnt/nfs        192.168.100.0/24(rw,sync,no_subtree_check)
 +
 +
sudo exportfs -a
 +
sudo modprobe xprtrdma
 +
sudo modprobe ib_ipoib
 +
echo "rdma 20049" | sudo tee -a /proc/fs/nfsd/portlist
 +
 +
</pre>
 +
 +
on client
 +
<pre>
 +
sudo modprobe xprtrdma
 +
sudo modprobe ib_ipoib
 +
sudo mount -o rdma,port=20049  10.1.100.1:/mnt/nfs /mnt/nfs
 +
</pre>
 +
 +
==LVM disk merged drive==
 +
follow instructions at https://www.digitalocean.com/community/tutorials/how-to-use-lvm-to-manage-storage-devices-on-ubuntu-18-04
 +
 +
<pre>
 +
sudo apt-get -y install lvm2
 +
sudo lvmdiskscan
 +
sudo pvcreate /dev/${DRIVE}
 +
sudo vgcreate nvmeg /dev/nvme2n1 /dev/nvme3n1 /dev/nvme4n1 /dev/nvme5n1 /dev/nvme0n1 /dev/nvme1n1
 +
sudo vgextend nvme4 /dev/${DRIVE}
 +
# create logical volume to take 100% space
 +
sudo lvcreate -l 100%FREE -n main nvmeg
 +
sudo fdisk -l
 +
# Disk /dev/mapper/nvmeg-main: 9.1 TiB, 10001997365248 bytes, 19535151104 sectors
 +
sudo lvmdiskscan
 +
sudo fdisk -l
 +
# extend volume and grow
 +
sudo lvextend -l +100%FREE /dev/mapper/nvme4-nvme4t
 +
sudo xfs_growfs -d /mnt/nvme1
 +
</pre>
 +
 +
== setup frp ==
 +
[[Setup FRP]]
 +
 +
== setup apache reverse proxy ==
 +
[[Setup apache2 reverse proxy]]
 +
 +
== VBoxManage ==
 +
[[VBoxManage]]
 +
== Github Action Runner ==
 +
[[Github Action Runner]]
 +
 +
== Squid https proxy ==
 +
[[Squid proxy docker]]
 +
 +
== pikvm ==
 +
enable oled and fan
 +
<pre>
 +
su
 +
rw
 +
systemctl enable --now kvmd-oled
 +
systemctl enable --now kvmd-fan
 +
# set admin login password
 +
kvmd-htpasswd set admin
 +
 +
ro
 +
</pre>
 +
 +
== update routing table on boot ==
 +
 +
<pre>
 +
echo "route del default wlp2s0" > /path/to/updateRoute.sh
 +
 +
sudo vim /etc/systemd/system/myrouting.service
 +
 +
[Unit]
 +
after=network
 +
 +
[Service]
 +
ExecStart=/path/to/updateRoute.sh
 +
 +
[Install]
 +
WantedBy=default.target
 +
 +
sudo systemctl enable myrouting.service
 +
 +
</pre>
 +
 +
 +
== set iterm2 terminal proxy==
 +
<pre>
 +
export http_proxy=socks5://127.0.0.1:8081
 +
export https_proxy=socks5://127.0.0.1:8081
 +
</pre>
 +
 +
== dell open manage OSMA install ==
 +
<pre>
 +
sudo apt install vim net-tools -y
 +
# choose one
 +
echo 'deb http://linux.dell.com/repo/community/openmanage/930/bionic bionic main' | sudo tee -a /etc/apt/sources.list.d/linux.dell.com.sources.list
 +
echo 'deb http://linux.dell.com/repo/community/openmanage/10300/focal focal main' | sudo tee -a /etc/apt/sources.list.d/linux.dell.com.sources.list
 +
echo 'deb http://linux.dell.com/repo/community/openmanage/11000/jammy jammy main' | sudo tee -a /etc/apt/sources.list.d/linux.dell.com.sources.list
 +
 +
wget https://linux.dell.com/repo/pgp_pubkeys/0x1285491434D8786F.asc
 +
sudo apt-key add 0x1285491434D8786F.asc
 +
sudo apt-get update
 +
sudo apt-get install srvadmin-all -y
 +
 +
 +
echo -e "\n${USER}    *    Administrator" | sudo tee -a /opt/dell/srvadmin/etc/omarolemap
 +
sudo service dsm_om_connsvc restart
 +
sudo systemctl start dsm_sa_datamgrd && sudo systemctl start dsm_sa_eventmgrd
 +
</pre>
 +
 +
== upgrade linux kernel to low latency ==
 +
<pre>
 +
sudo apt install linux-lowlatency
 +
 +
sudo grep menuentry /boot/grub/grub.cfg
 +
 +
sudo vim /etc/default/grub
 +
# change to
 +
GRUB_DEFAULT="gnulinux-5.15.0-79-lowlatency-advanced-4513eb34-58e6-408e-8ed7-3d487fe6b35b"
 +
GRUB_DISABLE_OS_PROBER=false
 +
 +
 +
sudo update-grub
 +
sudo reboot
 +
</pre>
 +
 +
== vnc server ==
 +
[[Vnc server | vnc server]]
 +
 +
== install openssl 3 on ubuntu ==
 +
<pre>
 +
sudo apt-get install make cmake
 +
 +
wget https://www.openssl.org/source/openssl-3.0.10.tar.gz
 +
tar -xzvf openssl-3.0.10.tar.gz
 +
cd /openssl-3.0.10
 +
./config
 +
make
 +
sudo make install
 +
sudo ldconfig
 +
</pre>
 +
 +
== install cuda 11.7 with pytorch ubuntu 22 ==
 +
<pre>
 +
sudo add-apt-repository ppa:graphics-drivers/ppa
 +
sudo apt update
 +
sudo apt-get remove --purge nvidia*
 +
sudo apt-get autoremove
 +
 +
sudo apt-get install nvidia-driver-535
 +
 +
sudo apt-key adv --fetch-keys  http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub
 +
sudo bash -c 'echo "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 /" > /etc/apt/sources.list.d/cuda.list'
 +
 +
sudo apt update
 +
 +
sudo apt-cache search cuda*
 +
sudo apt install cuda-11-8
 +
 +
pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
  
 +
# test pytorch cuda
 +
python
 +
import torch
 +
torch.cuda.is_available()
 
</pre>
 
</pre>

Latest revision as of 06:51, 24 June 2024

Contents

How to Install SVN on Ubuntu

http://www.howtogeek.com/howto/ubuntu/install-subversion-with-web-access-on-ubuntu/
http://forums.techarena.in/guides-tutorials/1209507.htm

RHEL Yum

AWS Redhat RHEL Install LAMP

Install VPN PPTPD on RHEL 7

Install OPen VPN client on ubuntu

#setup /etc/openvpn/openvpn.conf
#/etc/openvpn/auth.txt

#manually start
sudo openvpn --config /etc/openvpn/openvpn.conf &

AWS Redhat RHEL Install Ruby and Rails

Ubuntu Install GUI

clean up linux disk space

sudo journalctl --vacuum-time=2d
sudo apt-get autoremove --purge

sudo snap set system refresh.retain=2

tar and gzip

Archive a set of files:

tar -cvf tarfile.tar /var/log/syslog /var/log/messages

Archive and compress (gzip) a set of files:

tar -cvzf file.tar.gz /var/log/syslog /var/log/messages

Archive and compress (bzip2) a set of files:

tar -cvjf file.tar.bz2 /var/log/syslog /var/log/messages

Extract a tar file:

tar -xvf file.tar
tar -xvzf file.tar.gz
tar -xvjf file.tar.bz2

Display the content of a tar file:

tar -tvf file.tar
tar -tvzf file.tar.gz
tar -tvjf file.tar.bz2

Replace a file in an existing tar file:

tar -rvf tarfile.tar filetoreplace

Update a file in an existing tar file:

tar -uvf tarfile.tar newfile

Copy all files in one directory to another directory on local host:

cd /etc; tar cf – . | (cd /etc.bak; tar xvpf -)

#parallel zip
# compress
pigz -2 ./abc.vdi
# extract / decompress
pigz -d ./abc.vdi.gz

zip with split and password

interactive 

zip --encrypt -s 4g -r file.zip files

Or 

// recursive zip all folder
// with password
// split 4gb
zip --password password -s 4g -r output.zip input &
// with store only and 1gb split
zip -Z store -r -s 1g output.zip input/

no hangup even if logged out of ssh

nohup bash script.sh &> script.log &

DB backup

  1. mysqldump -u root -p [root_password] [tabdaase_name] > dumpfilename.sql
  2. mysqldump -u root -p password sugarcrm > sugarcrm.sql

//multiple

  1. mysqldump -u root -ptmppassword --databases bugs sugarcrm > bugs_sugarcrm.sql

//restore
mysql -u root -p[root_password] [database_name] < dumpfilename.sql

Cron job backup mysql databse every week

# Back up db. Run once a week at midnight on Sunday morning
0 0 * * 0 mysqldump -u root -pPASSWORD DB_NAME | gzip > /tmp/wiki_database_`date +'%Y-%m-%d'`.sql.gz

Search specific text among files

grep -H "textToFind" /var/www/*
grep -H "textToFind" /var/www/sites/*/html/.htaccess

Search file

find /home/david -name 'index*'

SVN cheat sheet

http://www.abbeyworkshop.com/howto/misc/svn01/
Set up svn in Ubuntu
http://www.howtogeek.com/howto/ubuntu/install-subversion-with-web-access-on-ubuntu/

Apache Rewrite

CodeIgniter Rewrite Rules 

<IfModule mod_rewrite.c>
	#Disable file listing
	Options -Indexes
    RewriteEngine On
    RewriteBase /
	#force www
	RewriteCond %{HTTP_HOST} ^hawkguide\.com [NC] 
	RewriteRule ^(.*) http://www.hawkguide.com/$1 [R=301,QSA,L] 
    #Removes access to the system folder by users.
    #Additionally this will allow you to create a System.php controller,
    #previously this would not have been possible.
    #'system' can be replaced if you have renamed your system folder.
    RewriteCond %{REQUEST_URI} ^system.*
    RewriteRule ^(.*)$ /index.php?/$1 [L]
    
    #When your application folder isn't in the system folder
    #This snippet prevents user access to the application folder
    #Submitted by: Fabdrol
    #Rename 'application' to your applications folder name.
    RewriteCond %{REQUEST_URI} ^application.*
    RewriteRule ^(.*)$ /index.php?/$1 [L]

    #Checks to see if the user is attempting to access a valid file,
    #such as an image or css document, if this isn't true it sends the
    #request to index.php
    RewriteCond %{REQUEST_FILENAME} !-f
    RewriteCond %{REQUEST_FILENAME} !-d
    RewriteRule ^(.*)$ index.php?/$1 [L]
</IfModule>

Create User

adduser newuser

Add a existing user to a group usermod -a -G www-data jerry

ImageMagick merge images

montage f*.jpg -tile 1x22 -geometry 1576x930+0+0 tile_.jpg


listen ports

 netstat -a | egrep 'Proto|LISTEN'

Disable prompt when ssh to unknown host

echo -e "StrictHostKeyChecking no\n" >> ~/.ssh/config

Apache server log permission denied

In CentOS
Permission denied: [client xx:xx] AH00132: file permissions deny server access: /var/www/ Run 

restorecon -R /var/www/foldername

squid 3.3 on centos

Configure SSL Bump on squid 3.3

List disk usage by directory

du -h <dir> | grep '[0-9\.]\+G'

Ubuntu create swap

sudo fallocate -l 1G /swapfile
ls -lh /swapfile
sudo chmod 600 /swapfile
ls -lh /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
sudo swapon --show

to auto mount 

echo '/swapfile none swap sw,nofail 0 0' | sudo tee -a /etc/fstab

https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-16-04

use entire disk for swap

sudo mkswap /dev/nvme4n1
sudo swapon /dev/nvme4n1

How to auto run scripts at system start in Cent OS 7

/etc/rc.local or /etc/rc.d/rc.local are no longer executed by default due to systemd-changes. to still use those, you need to make /etc/rc.d/rc.local executable: 

chmod +x /etc/rc.d/rc.local
sudo vim /etc/rc.d/rc.local
# then add yout commands to run

Mac fix mongodb max file limit issue / Adjust ulimit

echo kern.maxfiles=65536 | sudo tee -a /etc/sysctl.conf
echo kern.maxfilesperproc=65536 | sudo tee -a /etc/sysctl.conf
sudo sysctl -w kern.maxfiles=65536
sudo sysctl -w kern.maxfilesperproc=65536

Then set ulimit in bash_profile 

echo "ulimit -n 65536 65536" >> ~/.bash_profile

Ubuntu set timezone

sudo dpkg-reconfigure tzdata

Mac show git branch name in terminal

Add following code into ~/.bash_profile 

parse_git_branch() {
  git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/ (\1)/'
}

export PS1="\u@\h \W\[\033[32m\]\$(parse_git_branch)\[\033[00m\] $ "

Prepare extra ~/bin folder for shortcut commands

edit ~/.bash_profile add line 

export PATH="~/bin/:$PATH"

Auto generate git commit message with one file name

touch ~/bin/autocommit && chmod +x ~/bin/autocommit && vim ~/bin/autocommit

Paste 

#!/bin/bash
BRANCH_NAME=$(git symbolic-ref --short HEAD)
COMMIT_MSG=$(LANG=C git -c color.status=false status \
| sed -n -E -e '1,/Changes to be committed:/ d'\
            -e '1,1 d' \
            -e '/git reset/ d' \
            -e '/^Untracked files:/,$ d' \
            -e 's/^\s*//' \
            -e 's/      modified:   //' \
            -e '/./p')
FIRST_FILE=${COMMIT_MSG%%$'\n'*}
git commit -m "[${BRANCH_NAME}] ${FIRST_FILE}"

usage: just type autocommit and it will generate a commit message containing the current branch name and first file changed

Auto commit and push

git config --global push.default current
echo "autocommit && git push" > ~/bin/autopush
chmod +x ~/bin/autopush

usage: just type autopush

Mount hard drive

#find uuid
sudo blkid
#mount hard drive
sudo mount /dev/sda1 /mnt/ssd1


# auto mount
sudo vim /etc/fstab
#add 
UUID=0c18cc09-0028-4cb1-97ef-6222fe477432   /mnt/ssd   ext3    defaults,nofail     0        2

Better method 

DRIVE=sdb1
FOLDER=ssd1
UUID=$(sudo blkid /dev/${DRIVE} -sUUID -ovalue)
echo $UUID
echo "UUID=${UUID} /mnt/${FOLDER} xfs defaults,nofail 0 2" | sudo tee -a /etc/fstab
sudo mount --uuid $UUID /mnt/${FOLDER}

fix corrupt hard drive file system

sudo gdisk /dev/sdb

sudo e2fsck /dev/sdb1

using gparted for over 2tb

https://www.cyberciti.biz/tips/fdisk-unable-to-create-partition-greater-2tb.html 


sudo apt-get install -y gparted xfsprogs

DRIVE=sdb
FOLDER=ssd1

sudo parted /dev/${DRIVE}

mklabel gpt
mkpart primary 0G 100%
print
quit

sudo mkfs.xfs /dev/${DRIVE}1
sudo mkdir -p /mnt/${FOLDER}
# mount hard drive
UUID=$(sudo blkid /dev/${DRIVE}1 -sUUID -ovalue)
echo $UUID
echo "UUID=${UUID} /mnt/${FOLDER} xfs defaults,nofail 0 2" | sudo tee -a /etc/fstab
sudo mount --uuid $UUID /mnt/${FOLDER}

Test disk write speed

sudo chown -R $USER:$USER ./
dd if=/dev/zero of=./test1.img bs=1G count=1 oflag=dsync && rm ./test1.img

using fio 

sudo apt-get install fio


fio --filename=test --sync=1 --rw=randread --bs=4k --numjobs=1 \
  --iodepth=4 --group_reporting --name=test --filesize=2G --runtime=300 && rm test


fio --filename=test --sync=1 --rw=randwrite --bs=4k --numjobs=1 \
  --iodepth=4 --group_reporting --name=test --filesize=2G --runtime=300 && rm test

Splunk installation

download splunk then follow http://docs.splunk.com/Documentation/Splunk/7.0.3/Installation/InstallonLinux

Change port 

sudo vim $SPLUNK_HOME/etc/system/local/web.conf

add 

[settings]
httpport = 8866

Configure to start on boot time http://docs.splunk.com/Documentation/Splunk/latest/Admin/ConfigureSplunktostartatboottime 

sudo $SPLUNK_HOME/bin/splunk enable boot-start
sudo service splunk start

List file sizes by directory (depth = 1)

du -ah --max-depth=1 | sort -n

Copy files argument list too long

echo ./* | xargs cp -r --  /target-dir

echo ./* | xargs rsync -avh -- /target-dir

echo ./* | xargs  mv -t /target-dir --

clean up journal

sudo journalctl --vacuum-time=2days
sudo journalctl --vacuum-size=100M

sudo vim /etc/systemd/journald.conf
# change 
# SystemMaxUse=200M

sudo systemctl daemon-reload

https://ubuntuhandbook.org/index.php/2020/12/clear-systemd-journal-logs-ubuntu/

Allow pm2 run via ssh and cron

n=$(which node); \
n=${n%/bin/node}; \
chmod -R 755 $n/bin/*; \
sudo cp -r $n/{bin,lib,share} /usr/local

method 2: 

sudo rm -f /usr/local/bin/pm2 && sudo ln -s $(which pm2) /usr/local/bin/pm2

control cpu fan speed

see https://askubuntu.com/questions/22108/how-to-control-fan-speed 

sudo apt-get install fancontrol lm-sensors
sudo sensors-detect
# answer yes to all questions

check temp 

sensors

edit /etc/default/grub to add acpi_enforce_resources=lax to the GRUB_CMDLINE_LINUX_DEFAULT

then run 
sudo update-grub
reboot 
sudo service kmod start
sudo pwmconfig

view current config 

/etc/fancontrol

finally start fan control 

sudo service fancontrol start

auto start 

sudo update-rc.d fancontrol enable

config explaination

MINTEMP: The temperature (°C) at which to SHUT OFF the CPU fan. Efficient CPUs often will not need a fan while idling. Be sure to set this to a temperature that you know is safe. Setting this to 0 is not recommended and may ruin your hardware!

MAXTEMP: The temperature (°C) at which to spin the fan at its MAXIMUM speed. This should be probably be set to perhaps 10 or 20 degrees (°C) below your CPU's critical/shutdown temperature. Setting it closer to MINTEMP will result in higher fan speeds overall.

MINSTOP: The PWM value at which your fan stops spinning. Each fan is a little different. Power tweakers can echo different values (between 0 and 255) to /sys/class/hwmon/hwmon0/device/pwm1 and then watch the CPU fan. When the CPU fan stops, use this value.

MINSTART: The PWM value at which your fan starts to spin again. This is often a higher value than MINSTOP as more voltage is required to overcome inertia.

raw disk backup

if: input file

of: output file (image) 

dd if=/dev/mmcblk0 of=/media/usb500/pi_system_torrent.image

search for file name or extension

search for file name or extension find . -iname "*.jpg"

delete files by matches

find . -iname "*.jpg" -delete

rsync file from remote to local

nohup rsync -avh -e "ssh -x -T -o Compression=no" root@domain.com:/mnt/ssd1/* ./ >~/rsync.log &

The above command turn off compression so it will be faster.

rsync locally

sudo rsync -aqxP /mnt/ssd1/docker/ /nvmex4/docker

search

grep --color -n -C 1 "search_text" lib/npm.js

regex

grep --color -n -e "search_text[abc|abe]" lib/npm.js

search all files recursive

grep -HR --color "loan" ./

backup whole system

sudo su
dd if=/dev/sdc | bzip2 > /mnt/ssdbackup/ubuntuBackUp201803.bz2
bzcat /mnt/ssdbackup/ubuntuBackUp201803.bz2 | dd of=/dev/sdc

Grow linux partitions on live system

Start socks proxy server

see https://github.com/adegtyarev/docker-dante use docker-compose

Install fusion io drive2 driver in ubuntu

fusionio driver

Port Forwarding Simple TCP Traffic

use rinetd https://github.com/boutell/rinetd

rinetd simple tcp traffic forwarding

This is useful for forwarding traffic to server behind vpn

Auto restart process if exit

#/bin/bash
RIGID=$(hostname)
echo $RIGID
STATUS=`ps -A | grep gethbsc`
# change channel to slack webhook
CHANNEL="https://hooks.slack.com/services/xxxxxxxx/xxxxxxx"
if [ ! -z "$STATUS" -a "$STATUS" != " " ] ; then
  echo $STATUS
else
  echo "Restart service"
  cd ~/bsc && bash ~/bsc/start.sh
  curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"server=${RIGID} node down\"}" $CHANNEL
fi

RDMA NDS system via infiniband

this only works in centOS8

on server

change nfs disk array server ib ip to 10.1.100.1

change client server ib ip to 10.1.100.2 

vim /etc/exports
# edit and add
/mnt/nfs        10.1.100.0/24(rw,sync,no_subtree_check,insecure,no_root_squash)
/mnt/nfs        192.168.100.0/24(rw,sync,no_subtree_check)

sudo exportfs -a
sudo modprobe xprtrdma
sudo modprobe ib_ipoib
echo "rdma 20049" | sudo tee -a /proc/fs/nfsd/portlist

on client 

sudo modprobe xprtrdma
sudo modprobe ib_ipoib
sudo mount -o rdma,port=20049   10.1.100.1:/mnt/nfs /mnt/nfs

LVM disk merged drive

follow instructions at https://www.digitalocean.com/community/tutorials/how-to-use-lvm-to-manage-storage-devices-on-ubuntu-18-04 

sudo apt-get -y install lvm2
sudo lvmdiskscan
sudo pvcreate /dev/${DRIVE}
sudo vgcreate nvmeg /dev/nvme2n1 /dev/nvme3n1 /dev/nvme4n1 /dev/nvme5n1 /dev/nvme0n1 /dev/nvme1n1
sudo vgextend nvme4 /dev/${DRIVE}
# create logical volume to take 100% space
sudo lvcreate -l 100%FREE -n main nvmeg
sudo fdisk -l
# Disk /dev/mapper/nvmeg-main: 9.1 TiB, 10001997365248 bytes, 19535151104 sectors
sudo lvmdiskscan
sudo fdisk -l
# extend volume and grow
sudo lvextend -l +100%FREE /dev/mapper/nvme4-nvme4t
sudo xfs_growfs -d /mnt/nvme1

setup frp

Setup FRP

setup apache reverse proxy

Setup apache2 reverse proxy

VBoxManage

VBoxManage

Github Action Runner

Github Action Runner

Squid https proxy

Squid proxy docker

pikvm

enable oled and fan 

su
rw
systemctl enable --now kvmd-oled
systemctl enable --now kvmd-fan
# set admin login password
kvmd-htpasswd set admin

ro

update routing table on boot

echo "route del default wlp2s0" > /path/to/updateRoute.sh

sudo vim /etc/systemd/system/myrouting.service

[Unit]
after=network

[Service]
ExecStart=/path/to/updateRoute.sh

[Install]
WantedBy=default.target

sudo systemctl enable myrouting.service


set iterm2 terminal proxy

export http_proxy=socks5://127.0.0.1:8081
export https_proxy=socks5://127.0.0.1:8081

dell open manage OSMA install

sudo apt install vim net-tools -y
# choose one
echo 'deb http://linux.dell.com/repo/community/openmanage/930/bionic bionic main' | sudo tee -a /etc/apt/sources.list.d/linux.dell.com.sources.list
echo 'deb http://linux.dell.com/repo/community/openmanage/10300/focal focal main' | sudo tee -a /etc/apt/sources.list.d/linux.dell.com.sources.list
echo 'deb http://linux.dell.com/repo/community/openmanage/11000/jammy jammy main' | sudo tee -a /etc/apt/sources.list.d/linux.dell.com.sources.list

wget https://linux.dell.com/repo/pgp_pubkeys/0x1285491434D8786F.asc
sudo apt-key add 0x1285491434D8786F.asc
sudo apt-get update
sudo apt-get install srvadmin-all -y


echo -e "\n${USER}     *     Administrator" | sudo tee -a /opt/dell/srvadmin/etc/omarolemap
sudo service dsm_om_connsvc restart
sudo systemctl start dsm_sa_datamgrd && sudo systemctl start dsm_sa_eventmgrd

upgrade linux kernel to low latency

sudo apt install linux-lowlatency

sudo grep menuentry /boot/grub/grub.cfg

sudo vim /etc/default/grub
# change to 
GRUB_DEFAULT="gnulinux-5.15.0-79-lowlatency-advanced-4513eb34-58e6-408e-8ed7-3d487fe6b35b"
GRUB_DISABLE_OS_PROBER=false


sudo update-grub
sudo reboot

vnc server

vnc server

install openssl 3 on ubuntu

sudo apt-get install make cmake

wget https://www.openssl.org/source/openssl-3.0.10.tar.gz
tar -xzvf openssl-3.0.10.tar.gz
cd /openssl-3.0.10
./config
make
sudo make install
sudo ldconfig

install cuda 11.7 with pytorch ubuntu 22

sudo add-apt-repository ppa:graphics-drivers/ppa
sudo apt update
sudo apt-get remove --purge nvidia*
sudo apt-get autoremove

sudo apt-get install nvidia-driver-535

sudo apt-key adv --fetch-keys  http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub
sudo bash -c 'echo "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 /" > /etc/apt/sources.list.d/cuda.list'

sudo apt update

sudo apt-cache search cuda*
sudo apt install cuda-11-8

pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118

# test pytorch cuda
python
import torch
torch.cuda.is_available()
Retrieved from "http://wiki.hawkguide.com/index.php?title=Linux_Administration&oldid=742"