From 0cc6f1cc53a69993c3883afe74540328c28f7672 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 15:38:21 -0300 Subject: [PATCH 01/20] Source code modifications to make them compilable under Windows --- cmd/kmd/main.go | 6 +- cmd/kmd/mlock_windows.go | 24 ++++++ crypto/curve25519.go | 2 + crypto/libsodium-fork/configure.ac | 8 +- crypto/memcpy_chk_windows.c | 35 ++++++++ crypto/vrf.go | 2 + daemon/algod/api/server/router.go | 2 +- daemon/kmd/api/api.go | 2 +- libgoal/lockedFileWindows.go | 95 ++++++++++++++++++++++ logging/usage.go | 2 + logging/usage_windows.go | 65 +++++++++++++++ network/wsNetwork.go | 44 ---------- network/wsNetwork_common.go | 99 +++++++++++++++++++++++ network/wsNetwork_windows.go | 23 ++++++ nodecontrol/NodeController.go | 6 +- nodecontrol/kmdControl.go | 2 +- test/framework/fixtures/auctionFixture.go | 5 +- util/process_common.go | 27 +++++++ util/process_windows.go | 66 +++++++++++++++ util/util.go | 2 + util/util_windows.go | 28 +++++++ 21 files changed, 487 insertions(+), 58 deletions(-) create mode 100644 cmd/kmd/mlock_windows.go create mode 100644 crypto/memcpy_chk_windows.c create mode 100644 libgoal/lockedFileWindows.go create mode 100644 logging/usage_windows.go create mode 100644 network/wsNetwork_common.go create mode 100644 network/wsNetwork_windows.go create mode 100644 util/process_common.go create mode 100644 util/process_windows.go create mode 100644 util/util_windows.go diff --git a/cmd/kmd/main.go b/cmd/kmd/main.go index d08326f11d..752fa79070 100644 --- a/cmd/kmd/main.go +++ b/cmd/kmd/main.go @@ -21,11 +21,11 @@ import ( "os" "os/signal" "path/filepath" + "syscall" "time" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" - "golang.org/x/sys/unix" "github.com/algorand/go-algorand/cmd/kmd/codes" "github.com/algorand/go-algorand/daemon/kmd" @@ -93,8 +93,8 @@ func runKmd(dataDir string, timeoutSecs uint64) { // Timeouts can also send on the kill channel; because signal.Notify // will not block, this shouldn't cause an issue. From docs: "Package // signal will not block sending to c" - signal.Notify(kill, os.Interrupt, unix.SIGTERM, unix.SIGINT) - signal.Ignore(unix.SIGHUP) + signal.Notify(kill, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) + signal.Ignore(syscall.SIGHUP) // Build a kmd StartConfig startConfig := kmd.StartConfig{ diff --git a/cmd/kmd/mlock_windows.go b/cmd/kmd/mlock_windows.go new file mode 100644 index 0000000000..a15be39404 --- /dev/null +++ b/cmd/kmd/mlock_windows.go @@ -0,0 +1,24 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package main + +import ( + "github.com/algorand/go-algorand/logging" +) + +func tryMlockall(_ logging.Logger) { +} diff --git a/crypto/curve25519.go b/crypto/curve25519.go index 3f721c4328..1908976869 100644 --- a/crypto/curve25519.go +++ b/crypto/curve25519.go @@ -25,6 +25,8 @@ package crypto // #cgo linux,arm64 LDFLAGS: ${SRCDIR}/libs/linux/arm64/lib/libsodium.a // #cgo linux,arm CFLAGS: -I${SRCDIR}/libs/linux/arm/include // #cgo linux,arm LDFLAGS: ${SRCDIR}/libs/linux/arm/lib/libsodium.a +// #cgo windows,amd64 CFLAGS: -I${SRCDIR}/libs/windows/amd64/include +// #cgo windows,amd64 LDFLAGS: ${SRCDIR}/libs/windows/amd64/lib/libsodium.a // #include // #include "sodium.h" import "C" diff --git a/crypto/libsodium-fork/configure.ac b/crypto/libsodium-fork/configure.ac index 02619d4a13..8b53d33fe0 100644 --- a/crypto/libsodium-fork/configure.ac +++ b/crypto/libsodium-fork/configure.ac @@ -216,7 +216,7 @@ AC_CHECK_DEFINE([_FORTIFY_SOURCE], [], [ AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CFLAGS="$CFLAGS -fvisibility=hidden"]) -AS_CASE([$host_os], [cygwin*|mingw*|msys|pw32*|cegcc*], [ ], [ +AS_CASE([$host_os], [cygwin*|pw32*|mingw*|msys|cegcc*], [ ], [ AX_CHECK_COMPILE_FLAG([-fPIC], [CFLAGS="$CFLAGS -fPIC"]) ]) @@ -262,14 +262,14 @@ AC_ARG_ENABLE(soname-versions, ) AS_CASE([$host_os], - [cygwin*|mingw*|msys|pw32*|cegcc*], [ + [cygwin*|pw32*|mingw*|msys|cegcc*], [ AX_CHECK_LINK_FLAG([-Wl,--dynamicbase], [LDFLAGS="$LDFLAGS -Wl,--dynamicbase"]) AX_CHECK_LINK_FLAG([-Wl,--high-entropy-va], [LDFLAGS="$LDFLAGS -Wl,--high-entropy-va"]) AX_CHECK_LINK_FLAG([-Wl,--nxcompat], [LDFLAGS="$LDFLAGS -Wl,--nxcompat"]) ]) AS_CASE([$host_os], - [cygwin*|mingw*|msys|pw32*|cegcc*], [ + [cygwin*|pw32*|mingw*|msys|cegcc*], [ AX_CHECK_COMPILE_FLAG([-fno-asynchronous-unwind-tables], [ [CFLAGS="$CFLAGS -fno-asynchronous-unwind-tables"] ]) @@ -278,7 +278,7 @@ AS_CASE([$host_os], AS_IF([test "x$enable_ssp" != "xno"],[ AS_CASE([$host_os], - [cygwin*|mingw*|msys|pw32*|cegcc*|haiku], [ ], + [cygwin*|pw32*|mingw*|msys|cegcc*|haiku], [ ], [*], [ AX_CHECK_COMPILE_FLAG([-fstack-protector], [ AX_CHECK_LINK_FLAG([-fstack-protector], diff --git a/crypto/memcpy_chk_windows.c b/crypto/memcpy_chk_windows.c new file mode 100644 index 0000000000..a02ca32704 --- /dev/null +++ b/crypto/memcpy_chk_windows.c @@ -0,0 +1,35 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// +build gc + +#include "_cgo_export.h" +#include + +extern void crosscall2(void (*fn)(void *, int), void *, int); +extern void _cgo_panic(void *, int); + +void * __memcpy_chk (void *dstpp, const void *srcpp, size_t len, size_t dstlen) +{ + if (dstlen < len) { + struct { const char *p; } a; + + a.p = "panic from __memcpy_chk"; + crosscall2(_cgo_panic, &a, sizeof a); + *(int*)1 = 1; + } + return memcpy (dstpp, srcpp, len); +} diff --git a/crypto/vrf.go b/crypto/vrf.go index 25de87e7cb..4e1f549e1f 100644 --- a/crypto/vrf.go +++ b/crypto/vrf.go @@ -25,6 +25,8 @@ package crypto // #cgo linux,arm64 LDFLAGS: ${SRCDIR}/libs/linux/arm64/lib/libsodium.a // #cgo linux,arm CFLAGS: -I${SRCDIR}/libs/linux/arm/include // #cgo linux,arm LDFLAGS: ${SRCDIR}/libs/linux/arm/lib/libsodium.a +// #cgo windows,amd64 CFLAGS: -I${SRCDIR}/libs/windows/amd64/include +// #cgo windows,amd64 LDFLAGS: ${SRCDIR}/libs/windows/amd64/lib/libsodium.a // #include // #include "sodium.h" import "C" diff --git a/daemon/algod/api/server/router.go b/daemon/algod/api/server/router.go index f7cca11a87..9e48a893f4 100644 --- a/daemon/algod/api/server/router.go +++ b/daemon/algod/api/server/router.go @@ -56,7 +56,7 @@ // loader.Config.Import(), and that breaks the vendor directory if the source is symlinked from elsewhere) //go:generate swagger generate spec -o="../swagger.json" //go:generate swagger validate ../swagger.json --stop-on-error -//go:generate ./lib/bundle_swagger_json.sh +//go:generate sh ./lib/bundle_swagger_json.sh package server import ( diff --git a/daemon/kmd/api/api.go b/daemon/kmd/api/api.go index d38fb85adc..9b55951217 100644 --- a/daemon/kmd/api/api.go +++ b/daemon/kmd/api/api.go @@ -60,7 +60,7 @@ // loader.Config.Import(), and that breaks the vendor directory if the source is symlinked from elsewhere) //go:generate swagger generate spec -m -o="./swagger.json" //go:generate swagger validate ./swagger.json --stop-on-error -//go:generate ../lib/kmdapi/bundle_swagger_json.sh +//go:generate sh ../lib/kmdapi/bundle_swagger_json.sh package api import ( diff --git a/libgoal/lockedFileWindows.go b/libgoal/lockedFileWindows.go new file mode 100644 index 0000000000..13cb73008b --- /dev/null +++ b/libgoal/lockedFileWindows.go @@ -0,0 +1,95 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// +build windows + +package libgoal + +import ( + "errors" + "os" + "syscall" + "unsafe" +) + +type windowsLocker struct { +} + +var ( + kernel32, _ = syscall.LoadLibrary("kernel32.dll") + procLockFileEx, _ = syscall.GetProcAddress(kernel32, "LockFileEx") + procUnlockFileEx, _ = syscall.GetProcAddress(kernel32, "UnlockFileEx") +) + +const ( + winLockfileFailImmediately = 0x00000001 + winLockfileExclusiveLock = 0x00000002 + winLockfileSharedLock = 0x00000000 +) + +const ErrorLockViolation syscall.Errno = 0x21 // 33 + +// makeLocker create a windows file locker. +// note that the desired way is to use the OFD locker, which locks on the file descriptor level. +// falling back to the non-OFD lock would allow obtaining two locks by the same process. If this becomes +// and issue, we might want to use flock, which wouldn't work across NFS. +func makeLocker() (*windowsLocker, error) { + locker := &windowsLocker{} + return locker, nil +} + +func (f *windowsLocker) tryRLock(fd *os.File) error { + if errNo := lockFileEx(syscall.Handle(fd.Fd()), winLockfileSharedLock | winLockfileFailImmediately, 0, 1, 0, &syscall.Overlapped{}); errNo > 0 { + return errors.New("cannot lock file") + } + return nil +} + +func (f *windowsLocker) tryLock(fd *os.File) error { + if errNo := lockFileEx(syscall.Handle(fd.Fd()), winLockfileExclusiveLock | winLockfileFailImmediately, 0, 1, 0, &syscall.Overlapped{}); errNo > 0 { + return errors.New("cannot lock file") + } + return nil +} + +func (f *windowsLocker) unlock(fd *os.File) error { + if errNo := unlockFileEx(syscall.Handle(fd.Fd()), 0, 1, 0, &syscall.Overlapped{}); errNo > 0 { + return errors.New("cannot unlock file") + } + return nil +} + +func lockFileEx(handle syscall.Handle, flags uint32, reserved uint32, numberOfBytesToLockLow uint32, numberOfBytesToLockHigh uint32, offset *syscall.Overlapped) (syscall.Errno) { + r1, _, errNo := syscall.Syscall6(uintptr(procLockFileEx), 6, uintptr(handle), uintptr(flags), uintptr(reserved), uintptr(numberOfBytesToLockLow), uintptr(numberOfBytesToLockHigh), uintptr(unsafe.Pointer(offset))) + if r1 != 1 { + if errNo == 0 { + return syscall.EINVAL + } + return errNo + } + return 0 +} + +func unlockFileEx(handle syscall.Handle, reserved uint32, numberOfBytesToLockLow uint32, numberOfBytesToLockHigh uint32, offset *syscall.Overlapped) (syscall.Errno) { + r1, _, errNo := syscall.Syscall6(uintptr(procUnlockFileEx), 5, uintptr(handle), uintptr(reserved), uintptr(numberOfBytesToLockLow), uintptr(numberOfBytesToLockHigh), uintptr(unsafe.Pointer(offset)), 0) + if r1 != 1 { + if errNo == 0 { + return syscall.EINVAL + } + return errNo + } + return 0 +} diff --git a/logging/usage.go b/logging/usage.go index ec9e1fdd91..ed438cb43c 100644 --- a/logging/usage.go +++ b/logging/usage.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . +// +build !windows + package logging import ( diff --git a/logging/usage_windows.go b/logging/usage_windows.go new file mode 100644 index 0000000000..9978a3d8c1 --- /dev/null +++ b/logging/usage_windows.go @@ -0,0 +1,65 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package logging + +import ( + "context" + "sync" + "syscall" + "time" +) + +// UsageLogThread utility logging method +func UsageLogThread(ctx context.Context, log Logger, period time.Duration, wg *sync.WaitGroup) { + if wg != nil { + defer wg.Done() + } + var now time.Time + var prevTime time.Time + var Ktime, Utime syscall.Filetime + var prevKtime, prevUtime syscall.Filetime + ticker := time.NewTicker(period) + hasPrev := false + handle, err := syscall.GetCurrentProcess() + for true { + select { + case <-ticker.C: + case <-ctx.Done(): + return + } + now = time.Now() + if handle != 0 { + err = syscall.GetProcessTimes(handle, nil, nil, &Ktime, &Utime) + } else { + } + if err != nil { + } + if hasPrev { + userNanos := Utime.Nanoseconds() - prevUtime.Nanoseconds() + sysNanos := Ktime.Nanoseconds() - prevKtime.Nanoseconds() + wallNanos := now.Sub(prevTime).Nanoseconds() + userf := float64(userNanos) / float64(wallNanos) + sysf := float64(sysNanos) / float64(wallNanos) + log.Infof("usage nanos wall=%d user=%d sys=%d pu=%0.4f%% ps=%0.4f%%", wallNanos, userNanos, sysNanos, userf*100.0, sysf*100.0) + } else { + hasPrev = true + } + prevKtime = Ktime + prevUtime = Utime + prevTime = now + } +} diff --git a/network/wsNetwork.go b/network/wsNetwork.go index 5a154156ee..1a0b3c7a06 100644 --- a/network/wsNetwork.go +++ b/network/wsNetwork.go @@ -44,7 +44,6 @@ import ( "github.com/algorand/websocket" "github.com/gorilla/mux" "golang.org/x/net/netutil" - "golang.org/x/sys/unix" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" @@ -649,49 +648,6 @@ func (wn *WebsocketNetwork) setup() { } } -func (wn *WebsocketNetwork) rlimitIncomingConnections() error { - var lim unix.Rlimit - err := unix.Getrlimit(unix.RLIMIT_NOFILE, &lim) - if err != nil { - return err - } - - // If rlim_max is not sufficient, reduce IncomingConnectionsLimit - var rlimitMaxCap uint64 - if lim.Max < wn.config.ReservedFDs { - rlimitMaxCap = 0 - } else { - rlimitMaxCap = lim.Max - wn.config.ReservedFDs - } - if rlimitMaxCap > uint64(MaxInt) { - rlimitMaxCap = uint64(MaxInt) - } - if wn.config.IncomingConnectionsLimit > int(rlimitMaxCap) { - wn.log.Warnf("Reducing IncomingConnectionsLimit from %d to %d since RLIMIT_NOFILE is %d", - wn.config.IncomingConnectionsLimit, rlimitMaxCap, lim.Max) - wn.config.IncomingConnectionsLimit = int(rlimitMaxCap) - } - - // Set rlim_cur to match IncomingConnectionsLimit - newLimit := uint64(wn.config.IncomingConnectionsLimit) + wn.config.ReservedFDs - if newLimit > lim.Cur { - if runtime.GOOS == "darwin" && newLimit > 10240 && lim.Max == 0x7fffffffffffffff { - // The max file limit is 10240, even though - // the max returned by Getrlimit is 1<<63-1. - // This is OPEN_MAX in sys/syslimits.h. - // see https://github.com/golang/go/issues/30401 - newLimit = 10240 - } - lim.Cur = newLimit - err = unix.Setrlimit(unix.RLIMIT_NOFILE, &lim) - if err != nil { - return err - } - } - - return nil -} - // Start makes network connections and threads func (wn *WebsocketNetwork) Start() { var err error diff --git a/network/wsNetwork_common.go b/network/wsNetwork_common.go new file mode 100644 index 0000000000..059cd8a6bb --- /dev/null +++ b/network/wsNetwork_common.go @@ -0,0 +1,99 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// +build !windows + +package network + +import ( + "container/heap" + "context" + "encoding/base64" + "errors" + "fmt" + "io/ioutil" + "math" + "math/rand" + "net" + "net/http" + "net/url" + "path" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/algorand/go-deadlock" + "github.com/algorand/websocket" + "github.com/gorilla/mux" + "golang.org/x/net/netutil" + "golang.org/x/sys/unix" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/logging/telemetryspec" + "github.com/algorand/go-algorand/protocol" + tools_network "github.com/algorand/go-algorand/tools/network" + "github.com/algorand/go-algorand/util/metrics" +) + +func (wn *WebsocketNetwork) rlimitIncomingConnections() error { + var lim unix.Rlimit + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &lim) + if err != nil { + return err + } + + // If rlim_max is not sufficient, reduce IncomingConnectionsLimit + var rlimitMaxCap uint64 + if lim.Max < wn.config.ReservedFDs { + rlimitMaxCap = 0 + } else { + rlimitMaxCap = lim.Max - wn.config.ReservedFDs + } + if rlimitMaxCap > uint64(MaxInt) { + rlimitMaxCap = uint64(MaxInt) + } + if wn.config.IncomingConnectionsLimit > int(rlimitMaxCap) { + wn.log.Warnf("Reducing IncomingConnectionsLimit from %d to %d since RLIMIT_NOFILE is %d", + wn.config.IncomingConnectionsLimit, rlimitMaxCap, lim.Max) + wn.config.IncomingConnectionsLimit = int(rlimitMaxCap) + } + + // Set rlim_cur to match IncomingConnectionsLimit + newLimit := uint64(wn.config.IncomingConnectionsLimit) + wn.config.ReservedFDs + if newLimit > lim.Cur { + if runtime.GOOS == "darwin" && newLimit > 10240 && lim.Max == 0x7fffffffffffffff { + // The max file limit is 10240, even though + // the max returned by Getrlimit is 1<<63-1. + // This is OPEN_MAX in sys/syslimits.h. + // see https://github.com/golang/go/issues/30401 + newLimit = 10240 + } + lim.Cur = newLimit + err = unix.Setrlimit(unix.RLIMIT_NOFILE, &lim) + if err != nil { + return err + } + } + + return nil +} diff --git a/network/wsNetwork_windows.go b/network/wsNetwork_windows.go new file mode 100644 index 0000000000..af58d2cd00 --- /dev/null +++ b/network/wsNetwork_windows.go @@ -0,0 +1,23 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// +build windows + +package network + +func (wn *WebsocketNetwork) rlimitIncomingConnections() error { + return nil +} diff --git a/nodecontrol/NodeController.go b/nodecontrol/NodeController.go index 8467bf5ff6..3d775dfeb6 100644 --- a/nodecontrol/NodeController.go +++ b/nodecontrol/NodeController.go @@ -21,6 +21,8 @@ import ( "path/filepath" "syscall" "time" + + "github.com/algorand/go-algorand/util" ) // NodeController provides an object for controlling a specific algod node instance @@ -116,7 +118,7 @@ func killPID(pid int) error { return err } - err = syscall.Kill(pid, syscall.SIGTERM) + err = util.KillProcess(pid, syscall.SIGTERM) if err != nil { return err } @@ -129,7 +131,7 @@ func killPID(pid int) error { } select { case <-waitLong: - return syscall.Kill(pid, syscall.SIGKILL) + return util.KillProcess(pid, syscall.SIGTERM) case <-time.After(time.Millisecond * 100): } } diff --git a/nodecontrol/kmdControl.go b/nodecontrol/kmdControl.go index 00123add48..27b05dde79 100644 --- a/nodecontrol/kmdControl.go +++ b/nodecontrol/kmdControl.go @@ -172,7 +172,7 @@ func (kc *KMDController) StartKMD(args KMDStartArgs) (alreadyRunning bool, err e // Got a PID. Is there actually a process running there? // "If sig is 0, then no signal is sent, but existence and permission // checks are still performed" - err = syscall.Kill(int(pid), syscall.Signal(0)) + err = util.KillProcess(int(pid), syscall.Signal(0)) if err == nil { // Yup, return alreadyRunning = true return true, nil diff --git a/test/framework/fixtures/auctionFixture.go b/test/framework/fixtures/auctionFixture.go index b4e8251996..1d4b9b85c6 100644 --- a/test/framework/fixtures/auctionFixture.go +++ b/test/framework/fixtures/auctionFixture.go @@ -47,6 +47,7 @@ import ( "github.com/algorand/go-algorand/libgoal" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util" ) const ( @@ -221,7 +222,7 @@ func (f *AuctionFixture) Stop(pidFile string) error { return err } - err = syscall.Kill(int(pid), syscall.SIGTERM) + err = util.KillProcess(int(pid), syscall.SIGTERM) if err != nil { f.t.Errorf("Unable to kill PID: %d", pid) return err @@ -235,7 +236,7 @@ func (f *AuctionFixture) Stop(pidFile string) error { } select { case <-waitLong: - return syscall.Kill(int(pid), syscall.SIGKILL) + return util.KillProcess(int(pid), syscall.SIGKILL) case <-time.After(time.Millisecond * 100): } } diff --git a/util/process_common.go b/util/process_common.go new file mode 100644 index 0000000000..2ab8aff43b --- /dev/null +++ b/util/process_common.go @@ -0,0 +1,27 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// +build !windows + +package util + +import ( + "syscall" +) + +func KillProcess(pid int, sig Signal) error { + return syscall.Kill(pid, sig) +} diff --git a/util/process_windows.go b/util/process_windows.go new file mode 100644 index 0000000000..977047f196 --- /dev/null +++ b/util/process_windows.go @@ -0,0 +1,66 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// +build windows + +package util + +import ( + "os" + "unsafe" + + "golang.org/x/sys/windows" +) + +func KillProcess(pid int, _ os.Signal) error { + + p, err := os.FindProcess(pid) + if err == nil { + + for _, v := range getChildrenProcesses(pid){ + _ = v.Kill() + } + + err = p.Kill() + } + return err +} + +func getChildrenProcesses(parentPid int) ([]*os.Process) { + out := []*os.Process{} + snap, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, uint32(0)) + if err == nil { + var pe32 windows.ProcessEntry32 + + defer windows.CloseHandle(snap) + + pe32.Size = uint32(unsafe.Sizeof(pe32)) + if err := windows.Process32First(snap, &pe32); err == nil { + for { + if pe32.ParentProcessID == uint32(parentPid) { + p, err := os.FindProcess(int(pe32.ProcessID)) + if err == nil { + out = append(out, p) + } + } + if err = windows.Process32Next(snap, &pe32); err != nil { + break + } + } + } + } + return out +} diff --git a/util/util.go b/util/util.go index 2959e4904f..5e3b360424 100644 --- a/util/util.go +++ b/util/util.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . +// +build !windows + package util import ( diff --git a/util/util_windows.go b/util/util_windows.go new file mode 100644 index 0000000000..43b6563cf7 --- /dev/null +++ b/util/util_windows.go @@ -0,0 +1,28 @@ +// +build darwin freebsd netbsd openbsd + +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// +build !windows + +package util + +/* misc */ + +// RaiseRlimit increases the number of file descriptors we can have +func RaiseRlimit(_ uint64) error { + return nil +} From 12d0f0737c29ba4a0b4d10f8126b4513585b18bd Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 15:38:53 -0300 Subject: [PATCH 02/20] Scripts to setup development environment under MSYS2 --- Makefile | 19 ++++-- ...node_exporter-stable-windows-x86_64.tar.gz | Bin 0 -> 137 bytes scripts/build_package.sh | 7 ++- scripts/check_deps.sh | 7 ++- scripts/configure_dev.sh | 54 ++++++++++++++++++ scripts/dump_genesis.sh | 12 +++- scripts/ostype.sh | 2 + scripts/windows/install_deps.sh | 28 +++++++++ scripts/windows/install_shellcheck.sh | 28 +++++++++ scripts/windows/instructions.md | 25 ++++++++ 10 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 installer/external/node_exporter-stable-windows-x86_64.tar.gz create mode 100644 scripts/windows/install_deps.sh create mode 100644 scripts/windows/install_shellcheck.sh create mode 100644 scripts/windows/instructions.md diff --git a/Makefile b/Makefile index a5975b5d39..7d56ed613e 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,16 @@ -export GOPATH := $(shell go env GOPATH) -GOPATH1 := $(firstword $(subst :, ,$(GOPATH))) +UNAME := $(shell uname) +ifneq (, $(findstring MINGW,$(UNAME))) + #Gopath is not saved across sessions, probably existing Windows env vars, override them + export GOPATH := ${HOME}/go + GOPATH1 := $(GOPATH) + + export PATH := $(PATH):$(GOPATH)/bin +else + export GOPATH := $(shell go env GOPATH) + GOPATH1 := $(firstword $(subst :, ,$(GOPATH))) +endif export GO111MODULE := on export GOPROXY := direct - -UNAME := $(shell uname) SRCPATH := $(shell pwd) ARCH := $(shell ./scripts/archtype.sh) OS_TYPE := $(shell ./scripts/ostype.sh) @@ -38,6 +45,10 @@ endif endif endif +ifneq (, $(findstring MINGW,$(UNAME))) +EXTLDFLAGS := -static-libstdc++ -static-libgcc +endif + GOTAGS := --tags "$(GOTAGSLIST)" GOTRIMPATH := $(shell go help build | grep -q .-trimpath && echo -trimpath) diff --git a/installer/external/node_exporter-stable-windows-x86_64.tar.gz b/installer/external/node_exporter-stable-windows-x86_64.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..56d41a144d9e6f7471c3f870c43d82b4c459908e GIT binary patch literal 137 zcmb2|=3vOYUKGQ?{Pw&pU$X&ETVmfN;ed9F0>*cZ6F7pV2Q6)|u&dW~X5W_cesj&H zKfGoCDtKcv4t/dev/null) YELLOW_FG=$(tput setaf 3 2>/dev/null) END_FG_COLOR=$(tput sgr0 2>/dev/null) -GOPATH=$(go env GOPATH) +UNAME=$(uname) +if [[ "${UNAME}" == *"MINGW"* ]]; then + GOPATH=$HOME/go +else + GOPATH=$(go env GOPATH) +fi export GOPATH GO_BIN="$(echo "$GOPATH" | cut -d: -f1)/bin" MISSING=0 diff --git a/scripts/configure_dev.sh b/scripts/configure_dev.sh index cfb8757717..ea16f89653 100755 --- a/scripts/configure_dev.sh +++ b/scripts/configure_dev.sh @@ -41,6 +41,45 @@ function install_or_upgrade { fi } +function install_windoows_shellcheck() { + filename="shellcheck-v0.7.0.zip" + filechecksum="10ee2474845eeb76d8a13992457472b723edf470c7cf182a20b32ecee4ad009ec6b2ca542db8f66127cf19e24baf3a06838a0d101494a5a6c11b3b568f9f5a99" + wget https://shellcheck.storage.googleapis.com/$filename -O /tmp/$filename + if [ $? -ne 0 ] + then + rm /tmp/$filename &> /dev/null + echo "Error downloading $filename" + return 1 + fi + + if [ "$(cat /tmp/$filename | sha512sum | head -c 128)" != "$filechecksum" ] + then + rm /tmp/$filename &> /dev/null + echo "$filename checksum mismatch" + return 1 + fi + + unzip -o /tmp/shellcheck-v0.7.0.zip shellcheck-v0.7.0.exe -d /tmp + if [ $? -ne 0 ] + then + rm /tmp/$filename &> /dev/null + echo "Unable to decompress shellcheck file" + return 1 + fi + + mv -f /tmp/shellcheck-v0.7.0.exe /usr/bin/shellcheck.exe + if [ $? -ne 0 ] + then + rm /tmp/$filename &> /dev/null + echo "Unable to move shellcheck to /usr/bin" + exit 1 + fi + + rm /tmp/$filename &> /dev/null + + return 0 +} + if [ "${OS}" = "linux" ]; then if ! which sudo > /dev/null then @@ -61,6 +100,21 @@ elif [ "${OS}" = "darwin" ]; then install_or_upgrade automake install_or_upgrade shellcheck install_or_upgrade python3 +elif [ "${OS}" = "windows" ]; then + pacman -S --disable-download-timeout --noconfirm git automake autoconf m4 libtool mingw-w64-x86_64-python3 make mingw-w64-x86_64-gcc mingw-w64-x86_64-go mingw-w64-x86_64-boost mingw-w64-x86_64-python unzip procps + if [ $? -ne 0 ] + then + echo "Error installing pacman dependencies" + exit 1 + fi + + export GOPATH=$HOME/go + + install_windoows_shellcheck + if [ $? -ne 0 ] + then + exit 1 + fi fi if ${SKIP_GO_DEPS} ; then diff --git a/scripts/dump_genesis.sh b/scripts/dump_genesis.sh index c61d2fcbc7..de8705c352 100755 --- a/scripts/dump_genesis.sh +++ b/scripts/dump_genesis.sh @@ -9,13 +9,21 @@ D=$(mktemp -d) trap "rm -r $D" 0 GENJSON="$1" -GOPATH1=$(go env GOPATH | cut -d: -f1) +UNAME=$(uname) +if [[ "${UNAME}" == *"MINGW"* ]]; then + GOPATH1=$HOME/go +else + GOPATH1=$(go env GOPATH | cut -d: -f1) +fi $GOPATH1/bin/algod -d $D -g "$GENJSON" -x >/dev/null LEDGERS=$D/*/ledger.*sqlite for LEDGER in $LEDGERS; do for T in $(echo .tables | sqlite3 $LEDGER); do - case "$T" in + #remove trailing newlines echoed by Windows' sqlite3 app + T=${T//[$'\t\r\n ']} + + case $T in blocks) SORT=rnd ;; diff --git a/scripts/ostype.sh b/scripts/ostype.sh index 7d187f995e..44b14ca386 100755 --- a/scripts/ostype.sh +++ b/scripts/ostype.sh @@ -6,6 +6,8 @@ if [ "${UNAME}" = "Darwin" ]; then echo "darwin" elif [ "${UNAME}" = "Linux" ]; then echo "linux" +elif [[ "${UNAME}" == *"MINGW"* ]]; then + echo "windows" else echo "unsupported" exit 1 diff --git a/scripts/windows/install_deps.sh b/scripts/windows/install_deps.sh new file mode 100644 index 0000000000..56c54f03c2 --- /dev/null +++ b/scripts/windows/install_deps.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +pacman -S --disable-download-timeout --noconfirm git automake autoconf m4 libtool make mingw-w64-x86_64-gcc mingw-w64-x86_64-go mingw-w64-x86_64-boost mingw-w64-x86_64-python mingw-w64-x86_64-jq unzip procps +if [ $? -ne 0 ] +then + echo "Error installing pacman dependencies" + exit 1 +fi + +export GOPATH=$HOME/go + +# This is required because http://github.com/karalabe/hid library compiles with non-static libraries +cp /mingw64/bin/libwinpthread-1.dll $GOPATH/bin/ + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +$DIR/../configure_dev-deps.sh +if [ $? -ne 0 ] +then + exit 1 +fi + +$DIR/install_shellcheck.sh +if [ $? -ne 0 ] +then + exit 1 +fi + diff --git a/scripts/windows/install_shellcheck.sh b/scripts/windows/install_shellcheck.sh new file mode 100644 index 0000000000..32933e4d9a --- /dev/null +++ b/scripts/windows/install_shellcheck.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +version="v0.7.1" +wget https://github.com/koalaman/shellcheck/releases/download/$version/shellcheck-$version.zip -O /tmp/shellcheck-$version.zip +if [ $? -ne 0 ] +then + rm /tmp/shellcheck-$version.zip &> /dev/null + echo "Error downloading $filename" + exit 1 +fi + +unzip -o /tmp/shellcheck-$version.zip shellcheck-$version.exe -d /tmp +if [ $? -ne 0 ] +then + rm /tmp/shellcheck-$version.zip &> /dev/null + echo "Unable to decompress shellcheck file" + exit 1 +fi + +mv -f /tmp/shellcheck-$version.exe /usr/bin/shellcheck.exe +if [ $? -ne 0 ] +then + rm /tmp/shellcheck-$version.zip &> /dev/null + echo "Unable to move shellcheck to /usr/bin" + exit 1 +fi + +rm /tmp/shellcheck-$version.zip &> /dev/null diff --git a/scripts/windows/instructions.md b/scripts/windows/instructions.md new file mode 100644 index 0000000000..1e6ad04733 --- /dev/null +++ b/scripts/windows/instructions.md @@ -0,0 +1,25 @@ +1. Download and install `MSYS2` package from [here](https://www.msys2.org/) + +2. Run `MSYS2 MingW 64-bit` application to open the MSYS2 terminal. + +3. Update MSYS2 package and dependency manager by running the following commands: + + ``` + pacman -Syu --disable-download-timeout + ``` + + NOTE: It is very likely MSYS2 will ask to close the window and repeat the command for furter updates. Check `MSYS2` web page for additional support. + +4. Install GIT on MSYS2 by executing the following command: + + ``` + pacman -S --disable-download-timeout --noconfirm git + ``` + +5. Clone repository with `git clone https://github.com/algorand/go-algorand`. + +6. Switch to source code directory with `cd go-algorand`. + +7. Run `./scripts/windows/install_deps.sh` to install required dependencies. + +8. Run `make`. From f24f9312606cb9b03eaa95a8b1ff7ce381a4da2d Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 15:42:11 -0300 Subject: [PATCH 03/20] Moved dll dependency copy at the end of setup script --- scripts/windows/install_deps.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/windows/install_deps.sh b/scripts/windows/install_deps.sh index 56c54f03c2..610ef6b5be 100644 --- a/scripts/windows/install_deps.sh +++ b/scripts/windows/install_deps.sh @@ -9,9 +9,6 @@ fi export GOPATH=$HOME/go -# This is required because http://github.com/karalabe/hid library compiles with non-static libraries -cp /mingw64/bin/libwinpthread-1.dll $GOPATH/bin/ - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" $DIR/../configure_dev-deps.sh @@ -26,3 +23,5 @@ then exit 1 fi +# This is required because http://github.com/karalabe/hid library compiles with non-static libraries +cp /mingw64/bin/libwinpthread-1.dll $GOPATH/bin/ From 627e07da8b416c9624437d2c7d5514d0deb2d5d7 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 16:15:45 -0300 Subject: [PATCH 04/20] Fixed Linux compilation issues. --- network/wsNetwork_common.go | 31 ------------------------------- util/process_common.go | 2 +- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/network/wsNetwork_common.go b/network/wsNetwork_common.go index 059cd8a6bb..3cfce65b1c 100644 --- a/network/wsNetwork_common.go +++ b/network/wsNetwork_common.go @@ -19,40 +19,9 @@ package network import ( - "container/heap" - "context" - "encoding/base64" - "errors" - "fmt" - "io/ioutil" - "math" - "math/rand" - "net" - "net/http" - "net/url" - "path" - "regexp" "runtime" - "sort" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - "github.com/algorand/go-deadlock" - "github.com/algorand/websocket" - "github.com/gorilla/mux" - "golang.org/x/net/netutil" "golang.org/x/sys/unix" - - "github.com/algorand/go-algorand/config" - "github.com/algorand/go-algorand/crypto" - "github.com/algorand/go-algorand/logging" - "github.com/algorand/go-algorand/logging/telemetryspec" - "github.com/algorand/go-algorand/protocol" - tools_network "github.com/algorand/go-algorand/tools/network" - "github.com/algorand/go-algorand/util/metrics" ) func (wn *WebsocketNetwork) rlimitIncomingConnections() error { diff --git a/util/process_common.go b/util/process_common.go index 2ab8aff43b..bdedef9242 100644 --- a/util/process_common.go +++ b/util/process_common.go @@ -22,6 +22,6 @@ import ( "syscall" ) -func KillProcess(pid int, sig Signal) error { +func KillProcess(pid int, sig syscall.Signal) error { return syscall.Kill(pid, sig) } From 5eeb7f5cf12a61568cc14a03267470f12f02ffe5 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 19:28:58 -0300 Subject: [PATCH 05/20] Removed raise exception after cgo_painc call --- crypto/memcpy_chk_windows.c | 1 - 1 file changed, 1 deletion(-) diff --git a/crypto/memcpy_chk_windows.c b/crypto/memcpy_chk_windows.c index a02ca32704..cc7c67c45f 100644 --- a/crypto/memcpy_chk_windows.c +++ b/crypto/memcpy_chk_windows.c @@ -29,7 +29,6 @@ void * __memcpy_chk (void *dstpp, const void *srcpp, size_t len, size_t dstlen) a.p = "panic from __memcpy_chk"; crosscall2(_cgo_panic, &a, sizeof a); - *(int*)1 = 1; } return memcpy (dstpp, srcpp, len); } From fb401cbd1c9a37e74d713c52ed8413b2db26877f Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 19:29:25 -0300 Subject: [PATCH 06/20] Removed dummy comments --- libgoal/lockedFileWindows.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libgoal/lockedFileWindows.go b/libgoal/lockedFileWindows.go index 13cb73008b..79706615e6 100644 --- a/libgoal/lockedFileWindows.go +++ b/libgoal/lockedFileWindows.go @@ -40,12 +40,7 @@ const ( winLockfileSharedLock = 0x00000000 ) -const ErrorLockViolation syscall.Errno = 0x21 // 33 - // makeLocker create a windows file locker. -// note that the desired way is to use the OFD locker, which locks on the file descriptor level. -// falling back to the non-OFD lock would allow obtaining two locks by the same process. If this becomes -// and issue, we might want to use flock, which wouldn't work across NFS. func makeLocker() (*windowsLocker, error) { locker := &windowsLocker{} return locker, nil From df38db34d670ed31c335fc609e3ef1fc1a728648 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 19:30:01 -0300 Subject: [PATCH 07/20] Added comments to exported functions --- util/process_common.go | 1 + util/process_windows.go | 1 + 2 files changed, 2 insertions(+) diff --git a/util/process_common.go b/util/process_common.go index bdedef9242..ae74344470 100644 --- a/util/process_common.go +++ b/util/process_common.go @@ -22,6 +22,7 @@ import ( "syscall" ) +// KillProcess kills a running OS process func KillProcess(pid int, sig syscall.Signal) error { return syscall.Kill(pid, sig) } diff --git a/util/process_windows.go b/util/process_windows.go index 977047f196..baba740d9f 100644 --- a/util/process_windows.go +++ b/util/process_windows.go @@ -25,6 +25,7 @@ import ( "golang.org/x/sys/windows" ) +// KillProcess kills a running OS process func KillProcess(pid int, _ os.Signal) error { p, err := os.FindProcess(pid) From 8235d7c1cb42e3bdd4d561da1031e038afb75073 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 19:30:31 -0300 Subject: [PATCH 08/20] Fixed build exception on intended target OS --- util/util_windows.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/util/util_windows.go b/util/util_windows.go index 43b6563cf7..73df30495f 100644 --- a/util/util_windows.go +++ b/util/util_windows.go @@ -16,8 +16,6 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -// +build !windows - package util /* misc */ From e40994fb8512e7ec7aa7654c9ed3fea224dc0d1f Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 23:58:59 -0300 Subject: [PATCH 09/20] Added common functions for all OSes --- network/wsNetwork_test.go | 14 +++++---- util/util.go | 16 +++++++++++ util/util_windows.go | 60 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/network/wsNetwork_test.go b/network/wsNetwork_test.go index 40059ee1dd..584f77ba1e 100644 --- a/network/wsNetwork_test.go +++ b/network/wsNetwork_test.go @@ -30,7 +30,6 @@ import ( "strings" "sync" "sync/atomic" - "syscall" "testing" "time" @@ -43,6 +42,7 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util" "github.com/algorand/go-algorand/util/metrics" ) @@ -1069,14 +1069,16 @@ func TestWebsocketNetworkManyIdle(t *testing.T) { waitReady(t, clients[i], readyTimeout.C) } - var r0, r1 syscall.Rusage - syscall.Getrusage(syscall.RUSAGE_SELF, &r0) + var r0_utime, r1_utime int64 + var r0_stime, r1_stime int64 + + r0_utime, r0_stime = util.GetSystemTimes() time.Sleep(10 * time.Second) - syscall.Getrusage(syscall.RUSAGE_SELF, &r1) + r1_utime, r1_stime = util.GetSystemTimes() t.Logf("Background CPU use: user %v, system %v\n", - time.Duration(r1.Utime.Nano()-r0.Utime.Nano()), - time.Duration(r1.Stime.Nano()-r0.Stime.Nano())) + time.Duration(r1_utime-r0_utime), + time.Duration(r1_stime-r0_stime)) } // TODO: test both sides of http-header setting and checking? diff --git a/util/util.go b/util/util.go index 5e3b360424..93dc29bc6a 100644 --- a/util/util.go +++ b/util/util.go @@ -39,3 +39,19 @@ func RaiseRlimit(amount uint64) error { } return nil } + +// Getrusage gets file descriptors usage statistics +func Getrusage(who int, rusage *syscall.Rusage) (err error) { + err = syscall.Getrusage(who, rusage) +} + +// GetSystemTimes gets current OS kernel and usermode times +func GetSystemTimes() (utime int64, stime int64) { + var r syscall.Rusage + + syscall.Getrusage(syscall.RUSAGE_SELF, &r) + + utime = r.Utime.Nano() + stime = r.Stime.Nano() + return +} diff --git a/util/util_windows.go b/util/util_windows.go index 73df30495f..2fd4b9b46a 100644 --- a/util/util_windows.go +++ b/util/util_windows.go @@ -1,5 +1,3 @@ -// +build darwin freebsd netbsd openbsd - // Copyright (C) 2019-2020 Algorand, Inc. // This file is part of go-algorand // @@ -18,9 +16,67 @@ package util +import ( + "errors" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + + procGetSystemTimes = modkernel32.NewProc("GetSystemTimes") +) + /* misc */ // RaiseRlimit increases the number of file descriptors we can have func RaiseRlimit(_ uint64) error { return nil } + +// Getrusage gets file descriptors usage statistics +func Getrusage(who int, rusage *syscall.Rusage) (err error) { + if rusage != nil { + *rusage = syscall.Rusage{} + err = nil + } else { + err = errors.New("invalid parameter") + } + return +} + +// GetSystemTimes gets current OS kernel and usermode times +func GetSystemTimes() (utime int64, stime int64) { + var idleTime, kernelTime, userTime syscall.Filetime + + err := getSystemTimes(&idleTime, &kernelTime, &userTime) + if err == nil { + utime = filetimeToDuration(&userTime).Nanoseconds() + stime = filetimeToDuration(&kernelTime).Nanoseconds() + } else { + utime = 0 + stime = 0 + } + return +} + +func filetimeToDuration(ft *syscall.Filetime) time.Duration { + n := int64(ft.HighDateTime)<<32 + int64(ft.LowDateTime) // in 100-nanosecond intervals + return time.Duration(n * 100) +} + +func getSystemTimes(idleTime *syscall.Filetime, kernelTime *syscall.Filetime, userTime *syscall.Filetime) (err error) { + r1, _, e1 := syscall.Syscall(procGetSystemTimes.Addr(), 3, uintptr(unsafe.Pointer(idleTime)), uintptr(unsafe.Pointer(kernelTime)), uintptr(unsafe.Pointer(userTime))) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} From 71aa8ad5549a35d98e5c7d1eb0829d602bfb6a0f Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 22 Sep 2020 23:59:51 -0300 Subject: [PATCH 10/20] Fixed code formatting --- libgoal/lockedFileWindows.go | 8 ++++---- logging/usage.go | 2 +- util/process_windows.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libgoal/lockedFileWindows.go b/libgoal/lockedFileWindows.go index 79706615e6..3a33fba8f7 100644 --- a/libgoal/lockedFileWindows.go +++ b/libgoal/lockedFileWindows.go @@ -47,14 +47,14 @@ func makeLocker() (*windowsLocker, error) { } func (f *windowsLocker) tryRLock(fd *os.File) error { - if errNo := lockFileEx(syscall.Handle(fd.Fd()), winLockfileSharedLock | winLockfileFailImmediately, 0, 1, 0, &syscall.Overlapped{}); errNo > 0 { + if errNo := lockFileEx(syscall.Handle(fd.Fd()), winLockfileSharedLock|winLockfileFailImmediately, 0, 1, 0, &syscall.Overlapped{}); errNo > 0 { return errors.New("cannot lock file") } return nil } func (f *windowsLocker) tryLock(fd *os.File) error { - if errNo := lockFileEx(syscall.Handle(fd.Fd()), winLockfileExclusiveLock | winLockfileFailImmediately, 0, 1, 0, &syscall.Overlapped{}); errNo > 0 { + if errNo := lockFileEx(syscall.Handle(fd.Fd()), winLockfileExclusiveLock|winLockfileFailImmediately, 0, 1, 0, &syscall.Overlapped{}); errNo > 0 { return errors.New("cannot lock file") } return nil @@ -67,7 +67,7 @@ func (f *windowsLocker) unlock(fd *os.File) error { return nil } -func lockFileEx(handle syscall.Handle, flags uint32, reserved uint32, numberOfBytesToLockLow uint32, numberOfBytesToLockHigh uint32, offset *syscall.Overlapped) (syscall.Errno) { +func lockFileEx(handle syscall.Handle, flags uint32, reserved uint32, numberOfBytesToLockLow uint32, numberOfBytesToLockHigh uint32, offset *syscall.Overlapped) syscall.Errno { r1, _, errNo := syscall.Syscall6(uintptr(procLockFileEx), 6, uintptr(handle), uintptr(flags), uintptr(reserved), uintptr(numberOfBytesToLockLow), uintptr(numberOfBytesToLockHigh), uintptr(unsafe.Pointer(offset))) if r1 != 1 { if errNo == 0 { @@ -78,7 +78,7 @@ func lockFileEx(handle syscall.Handle, flags uint32, reserved uint32, numberOfBy return 0 } -func unlockFileEx(handle syscall.Handle, reserved uint32, numberOfBytesToLockLow uint32, numberOfBytesToLockHigh uint32, offset *syscall.Overlapped) (syscall.Errno) { +func unlockFileEx(handle syscall.Handle, reserved uint32, numberOfBytesToLockLow uint32, numberOfBytesToLockHigh uint32, offset *syscall.Overlapped) syscall.Errno { r1, _, errNo := syscall.Syscall6(uintptr(procUnlockFileEx), 5, uintptr(handle), uintptr(reserved), uintptr(numberOfBytesToLockLow), uintptr(numberOfBytesToLockHigh), uintptr(unsafe.Pointer(offset)), 0) if r1 != 1 { if errNo == 0 { diff --git a/logging/usage.go b/logging/usage.go index ed438cb43c..fba251bb9e 100644 --- a/logging/usage.go +++ b/logging/usage.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -// +build !windows +// +build !windows package logging diff --git a/util/process_windows.go b/util/process_windows.go index baba740d9f..9bb6703253 100644 --- a/util/process_windows.go +++ b/util/process_windows.go @@ -31,7 +31,7 @@ func KillProcess(pid int, _ os.Signal) error { p, err := os.FindProcess(pid) if err == nil { - for _, v := range getChildrenProcesses(pid){ + for _, v := range getChildrenProcesses(pid) { _ = v.Kill() } @@ -40,7 +40,7 @@ func KillProcess(pid int, _ os.Signal) error { return err } -func getChildrenProcesses(parentPid int) ([]*os.Process) { +func getChildrenProcesses(parentPid int) []*os.Process { out := []*os.Process{} snap, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, uint32(0)) if err == nil { From 981a7d5d58ee0826542901630cda8a7a3b650032 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Wed, 23 Sep 2020 00:09:26 -0300 Subject: [PATCH 11/20] Added missing return at end of function --- util/util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/util/util.go b/util/util.go index 93dc29bc6a..6714adc7bd 100644 --- a/util/util.go +++ b/util/util.go @@ -43,6 +43,7 @@ func RaiseRlimit(amount uint64) error { // Getrusage gets file descriptors usage statistics func Getrusage(who int, rusage *syscall.Rusage) (err error) { err = syscall.Getrusage(who, rusage) + return } // GetSystemTimes gets current OS kernel and usermode times From f6cd99087cf6edd2c54f23e2492739bb323f4400 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Wed, 23 Sep 2020 00:20:05 -0300 Subject: [PATCH 12/20] Removed underscore from variables --- network/wsNetwork_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/network/wsNetwork_test.go b/network/wsNetwork_test.go index 584f77ba1e..4cdce46c15 100644 --- a/network/wsNetwork_test.go +++ b/network/wsNetwork_test.go @@ -1069,16 +1069,16 @@ func TestWebsocketNetworkManyIdle(t *testing.T) { waitReady(t, clients[i], readyTimeout.C) } - var r0_utime, r1_utime int64 - var r0_stime, r1_stime int64 + var r0utime, r1utime int64 + var r0stime, r1stime int64 - r0_utime, r0_stime = util.GetSystemTimes() + r0utime, r0stime = util.GetSystemTimes() time.Sleep(10 * time.Second) - r1_utime, r1_stime = util.GetSystemTimes() + r1utime, r1stime = util.GetSystemTimes() t.Logf("Background CPU use: user %v, system %v\n", - time.Duration(r1_utime-r0_utime), - time.Duration(r1_stime-r0_stime)) + time.Duration(r1utime-r0utime), + time.Duration(r1stime-r0stime)) } // TODO: test both sides of http-header setting and checking? From 5f6538ef30eaf6775005a8c4a98237b4b1595a19 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Wed, 23 Sep 2020 00:43:09 -0300 Subject: [PATCH 13/20] Changed shell engine to newer bash --- scripts/dump_genesis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dump_genesis.sh b/scripts/dump_genesis.sh index de8705c352..b3c1f752b0 100755 --- a/scripts/dump_genesis.sh +++ b/scripts/dump_genesis.sh @@ -1,4 +1,4 @@ -#!/bin/sh -e +#!/usr/bin/env bash if [ "$1" = "" ]; then echo "Usage: $0 genesis.json" From ae02e07c264c2649b307108e863a1e76a2202efd Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Wed, 23 Sep 2020 13:47:42 -0300 Subject: [PATCH 14/20] Added a common function to get process running times. --- logging/usage.go | 38 +++++++++-------------- logging/usage_windows.go | 65 --------------------------------------- network/wsNetwork_test.go | 4 +-- util/util.go | 20 +++++++----- util/util_windows.go | 37 ++++++---------------- 5 files changed, 39 insertions(+), 125 deletions(-) delete mode 100644 logging/usage_windows.go diff --git a/logging/usage.go b/logging/usage.go index fba251bb9e..0d0515f8b6 100644 --- a/logging/usage.go +++ b/logging/usage.go @@ -14,53 +14,43 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -// +build !windows - package logging import ( "context" "sync" - "syscall" "time" -) -func timevalSubToMicroseconds(a, b syscall.Timeval) int64 { - seconds := a.Sec - b.Sec - var dusec int32 - if b.Usec > a.Usec { - seconds-- - dusec = int32(1000000) + int32(a.Usec-b.Usec) - } else { - dusec = int32(a.Usec - b.Usec) - } - return (int64(seconds) * 1000000) + int64(dusec) -} + "github.com/algorand/go-algorand/util" +) // UsageLogThread utility logging method func UsageLogThread(ctx context.Context, log Logger, period time.Duration, wg *sync.WaitGroup) { if wg != nil { defer wg.Done() } - var usage syscall.Rusage + var now time.Time - var prevUsage syscall.Rusage + var prevUtime, prevStime int64 + var Utime, Stime int64 var prevTime time.Time + ticker := time.NewTicker(period) hasPrev := false + for true { select { case <-ticker.C: case <-ctx.Done(): return } + now = time.Now() - err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage) - if err != nil { - } + Utime, Stime, _ = util.GetCurrentProcessTimes() + if hasPrev { - userNanos := timevalSubToMicroseconds(usage.Utime, prevUsage.Utime) * 1000 - sysNanos := timevalSubToMicroseconds(usage.Stime, prevUsage.Stime) * 1000 + userNanos := Utime - prevUtime + sysNanos := Stime - prevStime wallNanos := now.Sub(prevTime).Nanoseconds() userf := float64(userNanos) / float64(wallNanos) sysf := float64(sysNanos) / float64(wallNanos) @@ -68,7 +58,9 @@ func UsageLogThread(ctx context.Context, log Logger, period time.Duration, wg *s } else { hasPrev = true } - prevUsage = usage + + prevUtime = Utime + prevStime = Stime prevTime = now } } diff --git a/logging/usage_windows.go b/logging/usage_windows.go deleted file mode 100644 index 9978a3d8c1..0000000000 --- a/logging/usage_windows.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) 2019-2020 Algorand, Inc. -// This file is part of go-algorand -// -// go-algorand is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// go-algorand is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with go-algorand. If not, see . - -package logging - -import ( - "context" - "sync" - "syscall" - "time" -) - -// UsageLogThread utility logging method -func UsageLogThread(ctx context.Context, log Logger, period time.Duration, wg *sync.WaitGroup) { - if wg != nil { - defer wg.Done() - } - var now time.Time - var prevTime time.Time - var Ktime, Utime syscall.Filetime - var prevKtime, prevUtime syscall.Filetime - ticker := time.NewTicker(period) - hasPrev := false - handle, err := syscall.GetCurrentProcess() - for true { - select { - case <-ticker.C: - case <-ctx.Done(): - return - } - now = time.Now() - if handle != 0 { - err = syscall.GetProcessTimes(handle, nil, nil, &Ktime, &Utime) - } else { - } - if err != nil { - } - if hasPrev { - userNanos := Utime.Nanoseconds() - prevUtime.Nanoseconds() - sysNanos := Ktime.Nanoseconds() - prevKtime.Nanoseconds() - wallNanos := now.Sub(prevTime).Nanoseconds() - userf := float64(userNanos) / float64(wallNanos) - sysf := float64(sysNanos) / float64(wallNanos) - log.Infof("usage nanos wall=%d user=%d sys=%d pu=%0.4f%% ps=%0.4f%%", wallNanos, userNanos, sysNanos, userf*100.0, sysf*100.0) - } else { - hasPrev = true - } - prevKtime = Ktime - prevUtime = Utime - prevTime = now - } -} diff --git a/network/wsNetwork_test.go b/network/wsNetwork_test.go index 4cdce46c15..8137aa52a6 100644 --- a/network/wsNetwork_test.go +++ b/network/wsNetwork_test.go @@ -1072,9 +1072,9 @@ func TestWebsocketNetworkManyIdle(t *testing.T) { var r0utime, r1utime int64 var r0stime, r1stime int64 - r0utime, r0stime = util.GetSystemTimes() + r0utime, r0stime, _ = util.GetCurrentProcessTimes() time.Sleep(10 * time.Second) - r1utime, r1stime = util.GetSystemTimes() + r1utime, r1stime, _ = util.GetCurrentProcessTimes() t.Logf("Background CPU use: user %v, system %v\n", time.Duration(r1utime-r0utime), diff --git a/util/util.go b/util/util.go index 6714adc7bd..7dcb5ba310 100644 --- a/util/util.go +++ b/util/util.go @@ -46,13 +46,17 @@ func Getrusage(who int, rusage *syscall.Rusage) (err error) { return } -// GetSystemTimes gets current OS kernel and usermode times -func GetSystemTimes() (utime int64, stime int64) { - var r syscall.Rusage - - syscall.Getrusage(syscall.RUSAGE_SELF, &r) - - utime = r.Utime.Nano() - stime = r.Stime.Nano() +// GetCurrentProcessTimes gets current process kernel and usermode times +func GetCurrentProcessTimes() (utime int64, stime int64, err error) { + var usage syscall.Rusage + + err = syscall.Getrusage(syscall.RUSAGE_SELF, &usage) + if err == nil { + utime = usage.Utime.Nano() + stime = usage.Stime.Nano() + } else { + utime = 0 + stime = 0 + } return } diff --git a/util/util_windows.go b/util/util_windows.go index 2fd4b9b46a..a43de1e3be 100644 --- a/util/util_windows.go +++ b/util/util_windows.go @@ -20,15 +20,6 @@ import ( "errors" "syscall" "time" - "unsafe" - - "golang.org/x/sys/windows" -) - -var ( - modkernel32 = windows.NewLazySystemDLL("kernel32.dll") - - procGetSystemTimes = modkernel32.NewProc("GetSystemTimes") ) /* misc */ @@ -49,14 +40,18 @@ func Getrusage(who int, rusage *syscall.Rusage) (err error) { return } -// GetSystemTimes gets current OS kernel and usermode times -func GetSystemTimes() (utime int64, stime int64) { - var idleTime, kernelTime, userTime syscall.Filetime +// GetCurrentProcessTimes gets current process kernel and usermode times +func GetCurrentProcessTimes() (utime int64, stime int64, err error) { + var Ktime, Utime syscall.Filetime + var handle syscall.Handle - err := getSystemTimes(&idleTime, &kernelTime, &userTime) + handle, err = syscall.GetCurrentProcess() + if err == nil { + err = syscall.GetProcessTimes(handle, nil, nil, &Ktime, &Utime) + } if err == nil { - utime = filetimeToDuration(&userTime).Nanoseconds() - stime = filetimeToDuration(&kernelTime).Nanoseconds() + utime = filetimeToDuration(&Utime).Nanoseconds() + stime = filetimeToDuration(&Ktime).Nanoseconds() } else { utime = 0 stime = 0 @@ -68,15 +63,3 @@ func filetimeToDuration(ft *syscall.Filetime) time.Duration { n := int64(ft.HighDateTime)<<32 + int64(ft.LowDateTime) // in 100-nanosecond intervals return time.Duration(n * 100) } - -func getSystemTimes(idleTime *syscall.Filetime, kernelTime *syscall.Filetime, userTime *syscall.Filetime) (err error) { - r1, _, e1 := syscall.Syscall(procGetSystemTimes.Addr(), 3, uintptr(unsafe.Pointer(idleTime)), uintptr(unsafe.Pointer(kernelTime)), uintptr(unsafe.Pointer(userTime))) - if r1 == 0 { - if e1 != 0 { - err = error(e1) - } else { - err = syscall.EINVAL - } - } - return -} From 5dbcda342700b67cb1fc5ce6cafb4f562f26b200 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Wed, 23 Sep 2020 16:54:37 -0300 Subject: [PATCH 15/20] Added log and comment because of lack of mlockall functionality on Windows. --- cmd/kmd/mlock_windows.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/kmd/mlock_windows.go b/cmd/kmd/mlock_windows.go index a15be39404..f4f4dd896b 100644 --- a/cmd/kmd/mlock_windows.go +++ b/cmd/kmd/mlock_windows.go @@ -20,5 +20,10 @@ import ( "github.com/algorand/go-algorand/logging" ) -func tryMlockall(_ logging.Logger) { +// Windows does not have an mlockall functionality but might be emulated by +// calling SetProcessWorkingSetSize and VirtualLock like described here +// https://github.com/elastic/elasticsearch/pull/10887 but it can degrade +// OS performance. +func tryMlockall(log logging.Logger) { + log.Infof("running on windows -- mlockall not available") } From 08fb191eaec68978c936dd36d25a524eab21c447 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Mon, 28 Sep 2020 10:05:58 -0300 Subject: [PATCH 16/20] Removed indentation --- Makefile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 7d56ed613e..754d2128f2 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,12 @@ UNAME := $(shell uname) ifneq (, $(findstring MINGW,$(UNAME))) - #Gopath is not saved across sessions, probably existing Windows env vars, override them - export GOPATH := ${HOME}/go - GOPATH1 := $(GOPATH) - - export PATH := $(PATH):$(GOPATH)/bin +#Gopath is not saved across sessions, probably existing Windows env vars, override them +export GOPATH := ${HOME}/go +GOPATH1 := $(GOPATH) +export PATH := $(PATH):$(GOPATH)/bin else - export GOPATH := $(shell go env GOPATH) - GOPATH1 := $(firstword $(subst :, ,$(GOPATH))) +export GOPATH := $(shell go env GOPATH) +GOPATH1 := $(firstword $(subst :, ,$(GOPATH))) endif export GO111MODULE := on export GOPROXY := direct From eb48485e9776d9317212b22ab38b7386efda676f Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Mon, 28 Sep 2020 10:08:11 -0300 Subject: [PATCH 17/20] Merged Windows dependencies installer script with configure_dev.sh --- scripts/configure_dev.sh | 46 +++++++++++---------------- scripts/windows/install_deps.sh | 27 ---------------- scripts/windows/install_shellcheck.sh | 28 ---------------- 3 files changed, 19 insertions(+), 82 deletions(-) delete mode 100644 scripts/windows/install_deps.sh delete mode 100644 scripts/windows/install_shellcheck.sh diff --git a/scripts/configure_dev.sh b/scripts/configure_dev.sh index ea16f89653..8d69be65d9 100755 --- a/scripts/configure_dev.sh +++ b/scripts/configure_dev.sh @@ -42,40 +42,29 @@ function install_or_upgrade { } function install_windoows_shellcheck() { - filename="shellcheck-v0.7.0.zip" - filechecksum="10ee2474845eeb76d8a13992457472b723edf470c7cf182a20b32ecee4ad009ec6b2ca542db8f66127cf19e24baf3a06838a0d101494a5a6c11b3b568f9f5a99" - wget https://shellcheck.storage.googleapis.com/$filename -O /tmp/$filename - if [ $? -ne 0 ] - then - rm /tmp/$filename &> /dev/null - echo "Error downloading $filename" + version="v0.7.1" + wget https://github.com/koalaman/shellcheck/releases/download/$version/shellcheck-$version.zip -O /tmp/shellcheck-$version.zip + if [ $? -ne 0 ]; then + rm /tmp/shellcheck-$version.zip &> /dev/null + echo "Error downloading shellcheck $version" return 1 fi - if [ "$(cat /tmp/$filename | sha512sum | head -c 128)" != "$filechecksum" ] - then - rm /tmp/$filename &> /dev/null - echo "$filename checksum mismatch" - return 1 - fi - - unzip -o /tmp/shellcheck-v0.7.0.zip shellcheck-v0.7.0.exe -d /tmp - if [ $? -ne 0 ] - then - rm /tmp/$filename &> /dev/null - echo "Unable to decompress shellcheck file" + unzip -o /tmp/shellcheck-$version.zip shellcheck-$version.exe -d /tmp + if [ $? -ne 0 ]; then + rm /tmp/shellcheck-$version.zip &> /dev/null + echo "Unable to decompress shellcheck $version" return 1 fi - mv -f /tmp/shellcheck-v0.7.0.exe /usr/bin/shellcheck.exe - if [ $? -ne 0 ] - then - rm /tmp/$filename &> /dev/null + mv -f /tmp/shellcheck-$version.exe /usr/bin/shellcheck.exe + if [ $? -ne 0 ]; then + rm /tmp/shellcheck-$version.zip &> /dev/null echo "Unable to move shellcheck to /usr/bin" - exit 1 + return 1 fi - rm /tmp/$filename &> /dev/null + rm /tmp/shellcheck-$version.zip &> /dev/null return 0 } @@ -101,13 +90,14 @@ elif [ "${OS}" = "darwin" ]; then install_or_upgrade shellcheck install_or_upgrade python3 elif [ "${OS}" = "windows" ]; then - pacman -S --disable-download-timeout --noconfirm git automake autoconf m4 libtool mingw-w64-x86_64-python3 make mingw-w64-x86_64-gcc mingw-w64-x86_64-go mingw-w64-x86_64-boost mingw-w64-x86_64-python unzip procps + pacman -S --disable-download-timeout --noconfirm git automake autoconf m4 libtool make mingw-w64-x86_64-gcc mingw-w64-x86_64-go mingw-w64-x86_64-boost mingw-w64-x86_64-python mingw-w64-x86_64-jq unzip procps if [ $? -ne 0 ] then echo "Error installing pacman dependencies" exit 1 fi + # Golang probably is not installed under MSYS2 so add the environment variable temporarily export GOPATH=$HOME/go install_windoows_shellcheck @@ -115,6 +105,9 @@ elif [ "${OS}" = "windows" ]; then then exit 1 fi + + # This is required because http://github.com/karalabe/hid library compiles with non-static libraries + cp /mingw64/bin/libwinpthread-1.dll $GOPATH/bin/ fi if ${SKIP_GO_DEPS} ; then @@ -122,4 +115,3 @@ if ${SKIP_GO_DEPS} ; then fi "$SCRIPTPATH"/configure_dev-deps.sh - diff --git a/scripts/windows/install_deps.sh b/scripts/windows/install_deps.sh deleted file mode 100644 index 610ef6b5be..0000000000 --- a/scripts/windows/install_deps.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -pacman -S --disable-download-timeout --noconfirm git automake autoconf m4 libtool make mingw-w64-x86_64-gcc mingw-w64-x86_64-go mingw-w64-x86_64-boost mingw-w64-x86_64-python mingw-w64-x86_64-jq unzip procps -if [ $? -ne 0 ] -then - echo "Error installing pacman dependencies" - exit 1 -fi - -export GOPATH=$HOME/go - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -$DIR/../configure_dev-deps.sh -if [ $? -ne 0 ] -then - exit 1 -fi - -$DIR/install_shellcheck.sh -if [ $? -ne 0 ] -then - exit 1 -fi - -# This is required because http://github.com/karalabe/hid library compiles with non-static libraries -cp /mingw64/bin/libwinpthread-1.dll $GOPATH/bin/ diff --git a/scripts/windows/install_shellcheck.sh b/scripts/windows/install_shellcheck.sh deleted file mode 100644 index 32933e4d9a..0000000000 --- a/scripts/windows/install_shellcheck.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh - -version="v0.7.1" -wget https://github.com/koalaman/shellcheck/releases/download/$version/shellcheck-$version.zip -O /tmp/shellcheck-$version.zip -if [ $? -ne 0 ] -then - rm /tmp/shellcheck-$version.zip &> /dev/null - echo "Error downloading $filename" - exit 1 -fi - -unzip -o /tmp/shellcheck-$version.zip shellcheck-$version.exe -d /tmp -if [ $? -ne 0 ] -then - rm /tmp/shellcheck-$version.zip &> /dev/null - echo "Unable to decompress shellcheck file" - exit 1 -fi - -mv -f /tmp/shellcheck-$version.exe /usr/bin/shellcheck.exe -if [ $? -ne 0 ] -then - rm /tmp/shellcheck-$version.zip &> /dev/null - echo "Unable to move shellcheck to /usr/bin" - exit 1 -fi - -rm /tmp/shellcheck-$version.zip &> /dev/null From d4de8ab9747e511fa665f65c15c40682c9a7225c Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Mon, 28 Sep 2020 10:08:34 -0300 Subject: [PATCH 18/20] Updated Windows environment setup instructions --- scripts/windows/instructions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/windows/instructions.md b/scripts/windows/instructions.md index 1e6ad04733..b24370bcf7 100644 --- a/scripts/windows/instructions.md +++ b/scripts/windows/instructions.md @@ -20,6 +20,6 @@ 6. Switch to source code directory with `cd go-algorand`. -7. Run `./scripts/windows/install_deps.sh` to install required dependencies. +7. Run `./scripts/configure_dev.sh` to install required dependencies. 8. Run `make`. From 41087ef29b0fcb45e0fba80e823f7ee8926f57e6 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 29 Sep 2020 10:54:03 -0300 Subject: [PATCH 19/20] Fixed wrong change of SIGKILL signal to SIGTERM --- nodecontrol/NodeController.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodecontrol/NodeController.go b/nodecontrol/NodeController.go index 3d775dfeb6..62dfef7f87 100644 --- a/nodecontrol/NodeController.go +++ b/nodecontrol/NodeController.go @@ -131,7 +131,7 @@ func killPID(pid int) error { } select { case <-waitLong: - return util.KillProcess(pid, syscall.SIGTERM) + return util.KillProcess(pid, syscall.SIGKILL) case <-time.After(time.Millisecond * 100): } } From c1a10ecca5101b2aae6c9b8fbda85aa3a766d6a2 Mon Sep 17 00:00:00 2001 From: Mauro Leggieri Date: Tue, 29 Sep 2020 11:00:56 -0300 Subject: [PATCH 20/20] Fixed typo configure dev script --- scripts/configure_dev.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/configure_dev.sh b/scripts/configure_dev.sh index 8d69be65d9..0de2fed9f0 100755 --- a/scripts/configure_dev.sh +++ b/scripts/configure_dev.sh @@ -41,7 +41,7 @@ function install_or_upgrade { fi } -function install_windoows_shellcheck() { +function install_windows_shellcheck() { version="v0.7.1" wget https://github.com/koalaman/shellcheck/releases/download/$version/shellcheck-$version.zip -O /tmp/shellcheck-$version.zip if [ $? -ne 0 ]; then @@ -100,7 +100,7 @@ elif [ "${OS}" = "windows" ]; then # Golang probably is not installed under MSYS2 so add the environment variable temporarily export GOPATH=$HOME/go - install_windoows_shellcheck + install_windows_shellcheck if [ $? -ne 0 ] then exit 1