Using Go and implementations of database drivers using database/sql, the behavior that I appear to be experiencing with transactions appears to be that the connection needs to be closed after every transaction. If I don't, the database runs out of connections and I receive the following error :
"Begin Transaction failed. Error = Error 1040: Too many connections". This happens after 101 commits.
I have tried using two different drivers from github - lib/pq and go-sql-driver/mysql with the same results.
This behavior appears strange to me. Is this to be expected, or am I perhaps doing something incorrectly?
As requested, the code is below :
package main
import (
"database/sql"
"fmt"
"time"
"os"
"bufio"
"strconv"
_ "github.com/lib/pq"
//// _ "github.com/go-sql-driver/mysql"
//// _ "github.com/arnehormann/mysql"
)
const C_CONN_RDBMS = "postgres"
const C_CONN_STR = "user=admin dbname=testdb password=admin sslmode=disable"
////const C_CONN_RDBMS = "mysql"
////const C_CONN_STR = "test:test#/testdb?charset=utf8"
var pogDbConn *sql.DB // Db connection
func main() {
fmt.Println("\ntestdb1 - small test on "+C_CONN_RDBMS+" driver")
println()
var iIters int = fGetIterations()
println()
var tCloseConn bool = fGetCloseConn()
tmeStart := time.Now()
fDbTestInserts(iIters, tCloseConn) // run test Insert
fmt.Printf("Elapsed Time to process = %s\n", time.Since(tmeStart))
if pogDbConn != nil {
pogDbConn.Close()
}
}
func fDbTestInserts(iIters int, tCloseConn bool) {
var iCommitted int = 0
var oOsError error
var poDbTxn *sql.Tx
println("Running test inserts .........")
defer func() {fRollback(poDbTxn, iCommitted)} ()
for iPos := 1; iPos <= iIters; iPos += 1 {
if pogDbConn == nil { // must open db
pogDbConn, oOsError = sql.Open(C_CONN_RDBMS, C_CONN_STR)
if oOsError != nil {
fmt.Printf("Failed to open Db Connection. Error = %s\n")
return
}
}
poDbTxn, oOsError := pogDbConn.Begin()
if oOsError != nil {
fmt.Printf("Begin Transaction failed. Error = %s\n", oOsError)
return
}
var sSql string = "INSERT INTO test01 " +
"(sName, dBalance)" +
" VALUES ('Bart Simpson', 999.99)"
_, oOsError = poDbTxn.Exec(sSql)
if oOsError != nil {
fmt.Printf("INSERT for Table failed. Error = %s\n", oOsError)
return
}
_, oOsError = poDbTxn.Exec("COMMIT")
if oOsError != nil {
fmt.Printf("COMMIT for Insert failed. Error = %s\n", oOsError)
return
}
poDbTxn = nil
iCommitted += 1
if iPos%100 == 0 {
fmt.Printf("Iteration = %d, Inserted = %d \n", iPos, iCommitted)
}
if tCloseConn {
pogDbConn.Close()
pogDbConn = nil
}
}
fmt.Printf("Inserts completed - committed = %d\n", iCommitted)
}
func fRollback(poDbTxn *sql.Tx, iCommitted int) {
println("In fDbRollbackTran\n")
fmt.Printf("Committed trans = %d\n", iCommitted)
if poDbTxn == nil {
println("No Rollback required")
} else {
if pogDbConn == nil {
print ("Unable to Rollback - no connection")
} else {
println("Attempting Rollback")
var oOsError error = poDbTxn.Rollback()
if oOsError != nil {
fmt.Printf("Rollback of Transaction failed. Error = %s\n", oOsError)
} else {
println("Rollback Succeeded")
}
}
}
}
func fGetIterations() int {
oBufReader := bufio.NewReader(os.Stdin)
for {
print("Number of Inserts to process : (1 to 10,000) or 'end' : ")
vLine, _, _ := oBufReader.ReadLine()
var sInput string = string(vLine)
if sInput == "end" || sInput == "END" {
os.Exit(1)
}
iTot, oError := strconv.Atoi(sInput)
if oError != nil {
println("Invalid number")
} else if iTot < 1 || iTot > 10000 {
println ("Number must be from 1 to 10,000")
} else {
return iTot
}
}
}
func fGetCloseConn() bool {
oBufReader := bufio.NewReader(os.Stdin)
for {
print("Close Connection every transaction? (y/n/end) : ")
vLine, _, _ := oBufReader.ReadLine()
sInput := string(vLine)
if sInput == "y" || sInput == "n" {
return (sInput == "y")
}
if sInput == "end" || sInput == "END" {
os.Exit(1)
}
}
}
Commit should be done as follows (as advised to me) :
oOsError = poDbTxn.Commit()
Related
I am new to go, I am trying to get 3 functions to return them as follows
function 1 - To return memory usage of the system
function 2 - To return disk usage of the system
function 3 - To return CPU usage of the system
So far I am able to do this much only (PS: trying not to use any libs)
func getCPUTrack() (idle, total uint64) {
contents, err := ioutil.ReadFile("/proc/stat")
if err != nil {
return
}
lines := strings.Split(string(contents), "\n")
for _, line := range lines {
fields := strings.Fields(line)
if fields[0] == "cpu" {
numFields := len(fields)
for i := 1; i < numFields; i++ {
val, err := strconv.ParseUint(fields[i], 10, 64)
if err != nil {
fmt.Println("Error: ", i, fields[i], err)
}
total += val // tally up all the numbers to get total ticks
if i == 4 { // idle is the 5th field in the cpu line
idle = val
}
}
return
}
}
return
}
idle0, total0 := getCPUTrack()
time.Sleep(3 * time.Second)
idle1, total1 := getCPUTrack()
idleTicks := float64(idle1 - idle0)
totalTicks := float64(total1 - total0)
cpuUsage := 100 * (totalTicks - idleTicks) / totalTicks
fmt.Printf("CPU usage is %f%% [busy: %f, total: %f]\n", cpuUsage, totalTicks-idleTicks, totalTicks)
Can anyone help me with this?
Thanks
There is a pretty cool library you can use Go-osstat, or see in detail how it is implemented so you can build your own.
I've developed a client that uses this library and runs in the background sending Memory and CPU usage metrics
package main
import (
"fmt"
"os"
"time"
"github.com/mackerelio/go-osstat/cpu"
"github.com/mackerelio/go-osstat/memory"
)
const (
memoryMetric = "memory"
cpuMetric = "cpu"
retries = 10
)
type client struct {
packageName string
memoryIteration int
cpuIteration int
OSClient OSClient
}
type Client interface {
EmitMetrics()
}
type osClient struct{}
type OSClient interface {
GetMemory() (*memory.Stats, error)
GetCPU() (*cpu.Stats, error)
}
func (osc osClient) GetMemory() (*memory.Stats, error) { return memory.Get() }
func (osc osClient) GetCPU() (*cpu.Stats, error) { return cpu.Get() }
func NewClient(packageName string, memoryIteration, cpuIteration int) Client {
return newClient(packageName, memoryIteration, cpuIteration, osClient{})
}
func newClient(packageName string, memoryIteration, cpuIteration int, osclient OSClient) Client {
return &client{
packageName: packageName,
memoryIteration: memoryIteration,
cpuIteration: cpuIteration,
OSClient: osclient,
}
}
func (c *client) EmitMetrics() {
protectFromPanic := func(metric string) {
if r := recover(); r != nil {
fmt.Printf(fmt.Sprintf("Recover from fail sending %s metrics for %s", metric, c.packageName), zap.Any("recover", r))
}
}
c.sendMemoryMetrics(protectFromPanic)
c.sendCPUMetrics(protectFromPanic)
}
func (c *client) sendMemoryMetrics(f func(string)) {
count := 0
go func() {
defer func() {
f(memoryMetric)
}()
for {
memory, err := c.OSClient.GetMemory()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
count++
if count == retries {
return
}
} else {
count = 0
EmitMemoryMetrics(c.packageName, memory.Total, memory.Used, memory.Cached, memory.Free)
time.Sleep(time.Duration(c.memoryIteration) * time.Millisecond)
}
}
}()
}
func (c *client) sendCPUMetrics(f func(string)) {
go func() {
defer func() {
f(cpuMetric)
}()
for {
before, err := c.OSClient.GetCPU()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
}
time.Sleep(time.Duration(c.cpuIteration) * time.Millisecond)
after, err := c.OSClient.GetCPU()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
}
total := float64(after.Total - before.Total)
EmitCPUMetrics(c.packageName,
total,
float64(after.User-before.User)/total*100,
float64(after.System-before.System)/total*100,
float64(after.Idle-before.Idle)/total*100)
}
}()
}
I have a text file with this content:
192.168.1.2$nick
192.168.1.3$peter
192.168.1.4$mike
192.168.1.5$joe
A web server is running on each IP in the list.
I need to check if the servers are currently available and output a message if not available.
I wrote a small application. It works, but periodically produces incorrect results - it does not output messages for servers that are not actually available.
I can't figure out what's going on and in fact I'm not sure if I'm using http.Client correctly in goroutines.
Help me plese.
package main
import "fmt"
import "os"
import "strings"
import "io/ioutil"
import "net/http"
import "crypto/tls"
import "time"
import "strconv"
func makeGetRequest(URL string, c *http.Client) {
resp, err := c.Get(URL)
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
if !((resp.StatusCode >= 200 && resp.StatusCode <= 209)) {
fmt.Printf("%s-%d\n", URL, resp.StatusCode)
}
}
func makeHeadRequestAsync(tasks chan string, done chan bool, c *http.Client) {
for {
URL := <-tasks
if len(URL) == 0 {
break
}
resp, err := c.Head(URL)
if err != nil {
fmt.Println(err)
continue
}
defer resp.Body.Close()
if !((resp.StatusCode >= 200 && resp.StatusCode <= 209)) {
makeGetRequest(URL, c) // !!! Some servers do not support HEAD requests. !!!
}
}
done <- true
}
func main() {
if len(os.Args) < 3 {
fmt.Println("Usage: main <number of threads> <input-file>")
os.Exit(0)
}
threadsNum, err := strconv.Atoi(os.Args[1])
if err != nil {
fmt.Println("Bad first parameter. Exit.")
os.Exit(0)
}
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
client := &http.Client {
Timeout: 30 * time.Second,
}
file, err := ioutil.ReadFile(os.Args[2])
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fileLines := strings.Split(string(file), "\n")
tasks := make(chan string, threadsNum)
done := make(chan bool)
for i := 0; i < threadsNum; i++ {
go makeHeadRequestAsync(tasks, done, client)
}
for i := 0; i < len(fileLines); i++ {
tasks <- strings.Split(string(fileLines[i]), "$")[0:1][0]
}
for i := 0; i < threadsNum; i++ {
tasks <- ""
<-done
}
}
The program terminates when the main() function returns. The code does not ensure that all goroutines are done before returning from main.
Fix by doing the following:
Use a sync.WaitGroup to wait for the goroutines to complete before exiting the program.
Exit the goroutine when tasks is closed. Close tasks after submitting all work.
Here's the code:
func makeHeadRequestAsync(tasks chan string, wg *sync.WaitGroup, c *http.Client) {
defer wg.Done()
// for range on channel breaks when the channel is closed.
for URL := range tasks {
resp, err := c.Head(URL)
if err != nil {
fmt.Println(err)
continue
}
defer resp.Body.Close()
if !(resp.StatusCode >= 200 && resp.StatusCode <= 209) {
makeGetRequest(URL, c) // !!! Some servers do not support HEAD requests. !!!
}
}
}
func main() {
if len(os.Args) < 3 {
fmt.Println("Usage: main <number of threads> <input-file>")
os.Exit(0)
}
threadsNum, err := strconv.Atoi(os.Args[1])
if err != nil {
fmt.Println("Bad first parameter. Exit.")
os.Exit(0)
}
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
client := &http.Client{
Timeout: 30 * time.Second,
}
file, err := ioutil.ReadFile(os.Args[2])
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fileLines := strings.Split(string(file), "\n")
tasks := make(chan string)
var wg sync.WaitGroup
wg.Add(threadsNum)
for i := 0; i < threadsNum; i++ {
go makeHeadRequestAsync(tasks, &wg, client)
}
for i := 0; i < len(fileLines); i++ {
tasks <- strings.Split(string(fileLines[i]), "$")[0:1][0]
}
close(tasks)
wg.Wait()
}
This is my first Go program. I'm learning the language but it's a bit difficult to understand all the concepts so in order to practice I wrote a code to detect same file. It's a simple program which recursively check for duplicated files in a directory.
but:
how to detect duplicate file in directory files
the matter isn't directory recursively. the matter is how to compare
You could take the hash of each file body and then compare the hashes in a dictionary/map.
package main
import (
"crypto/md5"
"fmt"
"io"
"io/ioutil"
"log"
"os"
)
func main() {
contentHashes := make(map[string]string)
if err := readDir("./", contentHashes); err != nil {
log.Fatal(err)
}
}
func readDir(dirName string, contentHashes map[string]string) (err error) {
filesInfos, err := ioutil.ReadDir(dirName)
if err != nil {
return
}
for _, fi := range filesInfos {
if fi.IsDir() {
err := readDir(dirName+fi.Name()+"/", contentHashes)
if err != nil {
return err
}
} else {
// The important bits for this question
location := dirName + fi.Name()
// open the file
f, err := os.Open(location)
if err != nil {
return err
}
h := md5.New()
// copy the file body into the hash function
if _, err := io.Copy(h, f); err != nil {
return err
}
// Check if a file body with the same hash already exists
key := fmt.Sprintf("%x", h.Sum(nil))
if val, exists := contentHashes[key]; exists {
fmt.Println("Duplicate found", val, location)
} else {
contentHashes[key] = location
}
}
}
return
}
use sha256 to compare files
example:
package main
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"sync"
"flag"
"runtime"
"io"
)
var dir string
var workers int
type Result struct {
file string
sha256 [32]byte
}
func worker(input chan string, results chan<- *Result, wg *sync.WaitGroup) {
for file := range input {
var h = sha256.New()
var sum [32]byte
f, err := os.Open(file)
if err != nil {
fmt.Fprintln(os.Stderr, err)
continue
}
if _, err = io.Copy(h, f); err != nil {
fmt.Fprintln(os.Stderr, err)
f.Close()
continue
}
f.Close()
copy(sum[:], h.Sum(nil))
results <- &Result{
file: file,
sha256: sum,
}
}
wg.Done()
}
func search(input chan string) {
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
fmt.Fprintln(os.Stderr, err)
} else if info.Mode().IsRegular() {
input <- path
}
return nil
})
close(input)
}
func main() {
flag.StringVar(&dir, "dir", ".", "directory to search")
flag.IntVar(&workers, "workers", runtime.NumCPU(), "number of workers")
flag.Parse()
fmt.Printf("Searching in %s using %d workers...\n", dir, workers)
input := make(chan string)
results := make(chan *Result)
wg := sync.WaitGroup{}
wg.Add(workers)
for i := 0; i < workers; i++ {
go worker(input, results, &wg)
}
go search(input)
go func() {
wg.Wait()
close(results)
}()
counter := make(map[[32]byte][]string)
for result := range results {
counter[result.sha256] = append(counter[result.sha256], result.file)
}
for sha, files := range counter {
if len(files) > 1 {
fmt.Printf("Found %d duplicates for %s: \n", len(files), hex.EncodeToString(sha[:]))
for _, f := range files {
fmt.Println("-> ", f)
}
}
}
}
I want to read a text file character by character and print invalid input for those characters that exceed "H"and numbers that exceed "8".
eg: my input is
I9,A2
A10,C3
D2,L3
output:
invalid input for all three
package main
import (
"bufio"
"fmt"
"log"
"os"
)
func readLines(path string) ([]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
var lines []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
func main() {
lines, err := readLines("chessin.txt")
if err != nil {
log.Fatalf("readLines: %s", err)
}
var numLines int = len(lines)
for i := 0; i < numLines; i++ {
for j := 0; j < len(lines[i]); j++ {
if j > 'H' {
fmt.Printf("invalid input")
}
}
}
}
You need to edit the inner loop to check for every input line,
and find the number then comma and so, like this working sample code:
package main
import (
"bufio"
"fmt"
"log"
"os"
"strconv"
"strings"
)
func readLines(path string) ([]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
var lines []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
func checkOne(letter byte, number, i, j int) {
if letter > 'H' {
fmt.Printf("invalid input %q # (%d, %d) \n", letter, i, j)
}
if number > 8 {
fmt.Printf("invalid input number %d # (%d, %d) \n", number, i, j+1)
}
}
func main() {
lines, err := readLines("chessin.txt")
if err != nil {
log.Fatalf("readLines: %s", err)
}
var numLines int = len(lines)
for i := 0; i < numLines; i++ {
line := lines[i]
j := 0
comma := strings.IndexByte(line, ',')
if comma == -1 {
log.Fatalf("comma not found at line: %d", i)
}
number, err := strconv.Atoi(line[j+1 : comma])
if err != nil {
log.Fatalf("line:%d err: %s", i, err)
}
checkOne(line[j], number, i, j)
j = comma + 1
number, err = strconv.Atoi(line[j+1:])
if err != nil {
log.Fatalf("line:%d err: %s", i, err)
}
checkOne(line[j], number, i, j)
}
}
input file "chessin.txt":
I9,A2
A10,C3
D2,L3
output:
invalid input 'I' # (0, 0)
invalid input number 9 # (0, 1)
invalid input number 10 # (1, 1)
invalid input 'L' # (2, 3)
You can simplify by using string split and slice. Do not read and store all the values in an array and then loop through them again. Assuming the first character is always Caps and there will be no negative values. Working sample code is as follows:
package main
import (
"bufio"
"fmt"
"os"
"strconv"
"strings"
)
func main() {
file, err := os.Open("chessin.txt")
if err != nil {
fmt.Println(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
valid := []bool{}
for scanner.Scan() {
strs := strings.Split(scanner.Text(), ",")
valid = append(valid, validateStrings(strs))
}
fmt.Println(valid)
}
func validateStrings(strs []string) bool {
for _, str := range strs {
char := str[0]
num, err := strconv.Atoi(string(str[1:len(str)]))
if err != nil {
fmt.Println("Invalid input")
os.Exit(1)
}
if char < 'A' || char > 'H' || num > 8 || num < 0 {
return false
}
}
return true
}
How to rename a file in unicode format, the gaps?
When you try to rename the standard library function, the file is not located.
Old: /usr/home/spouk/go/src/simple/Agraba - I Know You Are Smiling Now (Original Mix).mp4 New: /usr/home/spouk/go/src/simple/Agraba_-_I_Know_You_Are_Smiling_Now__Original_Mix_.mp4 [RENAME] rename Agraba - I Know You Are Smiling Now (Original Mix).mp4 /usr/home/spouk/go/src/simple/Agraba_-_I_Know_You_Are_Smiling_Now__Original_Mix_.mp4: no such file or directory
Someone tell me what to use or where to get acquainted with the direction of solution to this problem. Tnx.
package main
import (
"flag"
d "github.com/fiam/gounidecode/unidecode"
"fmt"
"path/filepath"
"os"
"strings"
"strconv"
"os/exec"
)
type (
File struct {
OriginPath string
ConvertPath string
}
FilesStack struct {
Files []File
InPath string
OutPath string
}
)
func NewFilesStack() *FilesStack {
n := new(FilesStack)
n.Files = []File{}
return n
}
func (f *FilesStack) RuneToAscii(r rune) string {
if r < 128 {
return string(r)
} else {
return "\\u" + strconv.FormatInt(int64(r), 16)
}
}
func (f *FilesStack) TransliterCyr(str string) string {
var result []string
for _, sym := range d.Unidecode(str) {
if (f.RuneToAscii(sym) == " ") || (f.RuneToAscii(sym) == "'") || (f.RuneToAscii(sym) == "`") || (f.RuneToAscii(sym) == ")") || (f.RuneToAscii(sym) == "(") {
result = append(result, "_")
} else {
result = append(result, f.RuneToAscii(sym))
}
}
return strings.Join(result, "")
}
func (f *FilesStack) walkFunction(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
res, err := filepath.Abs(filepath.Dir(info.Name()))
if err != nil {
fmt.Printf(err.Error())
}
ext := filepath.Ext(info.Name())
if ext == ".mp4" {
file := new(File)
//make new file name
oldfile := filepath.Join(res, info.Name())
newname := filepath.Join(res, f.TransliterCyr(info.Name()))
file.OriginPath = filepath.Join(res, newname)
fmt.Printf("Old: %s\nNew: %s\n", oldfile, newname)
//rename file
if err_rename := os.Rename(info.Name(), newname); err_rename != nil {
fmt.Printf("[RENAME] %s\n", err_rename.Error())
}
tmpname := []string{}
name := strings.TrimRight(info.Name(), filepath.Ext(info.Name()))
for _, sym := range name {
if f.RuneToAscii(sym) != " " {
tmpname = append(tmpname, string(sym))
}
}
word := strings.Join(tmpname, "")
oo, _ := filepath.Abs(f.OutPath)
rr := strings.Join([]string{f.TransliterCyr(word), ".mp3"}, "")
//fmt.Printf("Res: %v %v\n", rr, word)
file.ConvertPath = filepath.Join(oo, rr)
//fmt.Printf("Infile: %v\n", file.OriginPath)
//fmt.Printf("Outfile: %v\n", file.ConvertPath)
f.Files = append(f.Files, *file)
}
}
return nil
}
func (f *FilesStack) ParseFilesmp4() {
//парсинг файлов
if err := filepath.Walk(f.InPath, f.walkFunction); err != nil {
fmt.Printf(err.Error())
}
//конвертация
for _, file := range f.Files {
fmt.Printf("Start convertation %s \n", file.OriginPath)
command := fmt.Sprintf("ffmpeg -i %s -f mp3 %s", file.OriginPath, file.ConvertPath)
//fmt.Printf("Command %v\n", command)
cmd, err_convert := exec.Command(command).Output()
if err_convert != nil {
fmt.Printf(err_convert.Error())
}
fmt.Printf("Result convert: %v\n", cmd)
}
}
// note, that variables are pointers
var (
inpath = flag.String("inpath", ".", "директория для поиска всех файлов с расширением .mp4")
outpath = flag.String("outpath", ".", "директория для сохранения .mp3")
)
func main() {
//make files structy
f := NewFilesStack()
//parse arguments
flag.Parse()
fmt.Printf("inpath: %s\noutpath: %s\n", *inpath, *outpath)
//set arguments
f.InPath = *inpath
f.OutPath = *outpath
f.ParseFilesmp4()
}